I know i should direct this at the creator of Sphider but things are slow there, its quite simple what i need to do in theory, currently the spider can have pages excluded from the index, which it noindex, no follows, i want to to noindex, follow.
Here’s the piece of code where (i think) it all happens:
[code=php] if (isset($_SERVER[‘argv’]) && $_SERVER[‘argc’] >= 2) {
$command_line = 1;
$ac = 1; //argument counter
while ($ac < (count($_SERVER[‘argv’]))) {
$arg = $_SERVER[‘argv’][$ac];
if ($arg == ‘-all’) {
$all = 1;
break;
} else if ($arg == ‘-u’) {
$url = $_SERVER[‘argv’][$ac+1];
$ac= $ac+2;
} else if ($arg == ‘-f’) {
$soption = ‘full’;
$ac++;
} else if ($arg == ‘-d’) {
$soption = ‘level’;
$maxlevel = $_SERVER[‘argv’][$ac+1];;
$ac= $ac+2;
} else if ($arg == ‘-l’) {
$domaincb = 1;
$ac++;
} else if ($arg == ‘-r’) {
$reindex = 1;
$ac++;
} else if ($arg == ‘-m’) {
$in = str_replace(“\n”, chr(10), $_SERVER[‘argv’][$ac+1]);
$ac= $ac+2;
} else if ($arg == ‘-n’) {
$out = str_replace(“\n”, chr(10), $_SERVER[‘argv’][$ac+1]);
$ac= $ac+2;
} else {
commandline_help();
die();
}
}
}