<html>

<head>
  <title>STEP-1</title>
</head>

<body>

<?php
require "./settings.inc";

$fp = @fopen("./sitemap.txt", "rt");
$fp2 = @fopen("./pages.txt", "w");
$ff=0;
$fff=0;
if ($fp) {
while(!feof($fp)) {
$url=trim(fgets($fp));
if($url!="") {
$nourl=nourl($url).".htm";
save_get_file($url, "./pages/".$nourl);
fputs($fp2,$nourl."\n");
$ff++;
$fff++;
echo "\n";
if($fff==rand(5,10)) { $fff=0; sleep(rand(0,3)); }
}
}
fclose($fp2);
fclose($fp);
}

$ff=0;
$fp = @fopen("./pages.txt", "rt");
$fp3 = @fopen("./processed.txt", "w");
if ($fp) {
while(!feof($fp)) {
$url=trim(fgets($fp));
if($url!="") {
$fp2 = fopen("./pages/".$url, "rt");
$cont=normalize(fread($fp2, filesize("./pages/".$url)));
fclose($fp2);
if($strip_before!="") {
unset($tmp);
$tmp=explode($strip_before,$cont);
array_shift($tmp);
$cont=$strip_before.implode($strip_before,$tmp);
}
if($strip_after!="") {
unset($tmp);
$tmp=explode($strip_after,$cont);
array_pop($tmp);
$cont=implode($strip_after,$tmp);
}
foreach($trans as $key => $val) {
$cont=str_replace($key,$val,$cont);
}
if($cont!="") {
$ff++;
$fp2 = @fopen("./processed/".$url, "w");
fputs($fp2,$cont);
fclose($fp2);
fputs($fp3,$url."\n");
}
echo "\n";
}
}
fclose($fp3);
fclose($fp);
}
echo "<h2>Processed pages: $ff</h2>";
?>
<h1>OK</h1>
</body>

</html>