<?php
// ** SETTINGS ** /
ini_set('memory_limit', '6000M');
$find              = '/home/new_parser/200pairs.data*';
$save              = '/home/new_parser/new_200pairs_btc-xrp.csv';
$marketname        = array('BTC-XRP');
$linesPerIteration = 7000000;
// ** SETTINGS END ** /
$starttime    = time();
$lastprogress = false;
$files        = glob($find);
natsort($files);
foreach($files as $file){
   if (file_exists($file)){
      $i=0;
      $totalLines = (int)trim(shell_exec("wc -l ".$file));
      progress($file, $totalLines, $i*$linesPerIteration);
      while(true){
         //progress($file, $totalLines, $i*$linesPerIteration);
         $lines = explode("\n", trim(shell_exec("sed -n '".(1+$i*$linesPerIteration).",".($linesPerIteration+$i*$linesPerIteration)."p' ".$file)));
         $originalLineCount = count($lines);
         foreach($lines as $lineID => $line){
            if ((strpos($line, $marketname[0]) !== false) && (strpos($line, "'Fills': [{") !== false)) {
               $line = str_replace("'MarketName': None", "'MarketName': null", $line);
               $line = str_replace("'", '"', $line);
               $json = json_decode($line, true);
               if ($json && $json['Fills'] && in_array(trim($json['MarketName']), $marketname)) {
                  foreach($json['Fills'] as $b){
                     $ms = explode('.',$b['TimeStamp']);
                     if (!isset($ms[1])){ $ms[1]=0; }
                     $ms=str_pad($ms[1],3,'0');
                     $data  = strtotime($b['TimeStamp']).$ms;
                     $data .= ',';
                     $data .= $b['OrderType'];
                     $data .= ',';
                     $data .= (isset($b['Rate'])?number_format($b['Rate'],12,'.',''):'null');
                     $data .= ',';
                     $data .= $b['Quantity'];
                     $data .= "\n";
                     file_put_contents($save, $data, FILE_APPEND );
                  }
               }
            }
            unset($lines[$lineID]);
         }
         if ($originalLineCount<=1){ break; }
         $i++;
      }
      progress($file, $totalLines, $i*$linesPerIteration);
   }
   else{
      die('File does not exist.');
   }
}
function progress($file, $lines, $doneLines){
   global $starttime;
   global $lastprogress;
   if ($lastprogress === false || time()-$lastprogress>1){
      error_log(time()-$starttime.
         ' seconds in file '.$file.
         ', '.($doneLines).'/'.
         ($lines).
         ' ['.round($doneLines/$lines*100).'%]');
      $lastprogress = time();
   }
}
?>