Untitled
5 years ago in Plain Text
<?php
// ** SETTINGS ** /
ini_set('memory_limit', '6000M');
$find = '/home/new_parser/200pairs.data*';
$save = '/home/new_parser/new_200pairs_btc-xrp.csv';
$marketname = array('BTC-XRP');
$linesPerIteration = 7000000;
// ** SETTINGS END ** /
$starttime = time();
$lastprogress = false;
$files = glob($find);
natsort($files);
foreach($files as $file){
if (file_exists($file)){
$i=0;
$totalLines = (int)trim(shell_exec("wc -l ".$file));
progress($file, $totalLines, $i*$linesPerIteration);
while(true){
//progress($file, $totalLines, $i*$linesPerIteration);
$lines = explode("\n", trim(shell_exec("sed -n '".(1+$i*$linesPerIteration).",".($linesPerIteration+$i*$linesPerIteration)."p' ".$file)));
$originalLineCount = count($lines);
foreach($lines as $lineID => $line){
if ((strpos($line, $marketname[0]) !== false) && (strpos($line, "'Fills': [{") !== false)) {
$line = str_replace("'MarketName': None", "'MarketName': null", $line);
$line = str_replace("'", '"', $line);
$json = json_decode($line, true);
if ($json && $json['Fills'] && in_array(trim($json['MarketName']), $marketname)) {
foreach($json['Fills'] as $b){
$ms = explode('.',$b['TimeStamp']);
if (!isset($ms[1])){ $ms[1]=0; }
$ms=str_pad($ms[1],3,'0');
$data = strtotime($b['TimeStamp']).$ms;
$data .= ',';
$data .= $b['OrderType'];
$data .= ',';
$data .= (isset($b['Rate'])?number_format($b['Rate'],12,'.',''):'null');
$data .= ',';
$data .= $b['Quantity'];
$data .= "\n";
file_put_contents($save, $data, FILE_APPEND );
}
}
}
unset($lines[$lineID]);
}
if ($originalLineCount<=1){ break; }
$i++;
}
progress($file, $totalLines, $i*$linesPerIteration);
}
else{
die('File does not exist.');
}
}
function progress($file, $lines, $doneLines){
global $starttime;
global $lastprogress;
if ($lastprogress === false || time()-$lastprogress>1){
error_log(time()-$starttime.
' seconds in file '.$file.
', '.($doneLines).'/'.
($lines).
' ['.round($doneLines/$lines*100).'%]');
$lastprogress = time();
}
}
?>