Untitled
1 week ago in Plain Text
<?php

// ** SETTINGS ** /

ini_set('memory_limit', '6000M');
$find = '/home/new_parser/200pairs.data*';
$save = '/home/new_parser/new_200pairs_btc-xrp.csv';
$marketname = array('BTC-XRP');
$linesPerIteration = 7000000;

// ** SETTINGS END ** /

$starttime = time();
$lastprogress = false;
$files = glob($find);
natsort($files);

foreach($files as $file){
if (file_exists($file)){
$i=0;
$totalLines = (int)trim(shell_exec("wc -l ".$file));
progress($file, $totalLines, $i*$linesPerIteration);
while(true){
//progress($file, $totalLines, $i*$linesPerIteration);
$lines = explode("\n", trim(shell_exec("sed -n '".(1+$i*$linesPerIteration).",".($linesPerIteration+$i*$linesPerIteration)."p' ".$file)));
$originalLineCount = count($lines);
foreach($lines as $lineID => $line){
if ((strpos($line, $marketname[0]) !== false) && (strpos($line, "'Fills': [{") !== false)) {
$line = str_replace("'MarketName': None", "'MarketName': null", $line);
$line = str_replace("'", '"', $line);
$json = json_decode($line, true);
if ($json && $json['Fills'] && in_array(trim($json['MarketName']), $marketname)) {
foreach($json['Fills'] as $b){
$ms = explode('.',$b['TimeStamp']);
if (!isset($ms[1])){ $ms[1]=0; }
$ms=str_pad($ms[1],3,'0');

$data = strtotime($b['TimeStamp']).$ms;
$data .= ',';
$data .= $b['OrderType'];
$data .= ',';
$data .= (isset($b['Rate'])?number_format($b['Rate'],12,'.',''):'null');
$data .= ',';
$data .= $b['Quantity'];
$data .= "\n";

file_put_contents($save, $data, FILE_APPEND );

}
}
}
unset($lines[$lineID]);
}
if ($originalLineCount<=1){ break; }
$i++;
}
progress($file, $totalLines, $i*$linesPerIteration);
}
else{
die('File does not exist.');
}
}

function progress($file, $lines, $doneLines){
global $starttime;
global $lastprogress;
if ($lastprogress === false || time()-$lastprogress>1){
error_log(time()-$starttime.
' seconds in file '.$file.
', '.($doneLines).'/'.
($lines).
' ['.round($doneLines/$lines*100).'%]');
$lastprogress = time();
}
}

?>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78