data-pipelines/src/munge.rs

#![allow(unused)]

#[macro_use]
extern crate slog;
#[macro_use]
extern crate markets;

use std::io::{self, prelude::*};
use std::fs;
use std::path::{Path, PathBuf};
use std::time::*;
use pretty_toa::ThousandsSep;
use structopt::StructOpt;
use serde::{Serialize, Deserialize};
use slog::Drain;
use chrono::{DateTime, Utc};
use markets::crypto::{Exchange, Ticker, Side};

macro_rules! fatal { ($fmt:expr, $($args:tt)*) => {{
    eprintln!($fmt, $($args)*);
    std::process::exit(1);
}}}



#[structopt(rename_all="kebab-case")]
#[derive(Debug, StructOpt)]
enum Opt {

    /// Filter trades-csv by start,end range and save subset to output-path
    Range {

        /// Path to CSV file with trades data
        #[structopt(short = "f", long = "trades-csv")]
        #[structopt(parse(from_os_str))]
        trades_csv: PathBuf,

        /// Where to save the query results (CSV output)
        #[structopt(short = "o", long = "output-path")]
        #[structopt(parse(from_os_str))]
        output_path: PathBuf,

        /// rfc3339 format ("YYYY-MM-DDTHH:MM:SSZ")
        start: DateTime<Utc>,

        /// rfc3339 format ("YYYY-MM-DDTHH:MM:SSZ")
        end: DateTime<Utc>,

    },
}

#[derive(Deserialize)]
struct Trade {
    /// Unix nanoseconds
    pub time: u64,
    pub exch: Exchange,
    pub ticker: Ticker,
    //pub side: Option<Side>,
    pub price: f64,
    pub amount: f64,
}

fn per_sec(n: usize, span: Duration) -> f64 {
    if n == 0 || span < Duration::from_micros(1) { return 0.0 }
    let s: f64 = span.as_nanos() as f64 / 1e9f64;
    n as f64 / s
}

fn nanos(utc: DateTime<Utc>) -> u64 {
    (utc.timestamp() as u64) * 1_000_000_000_u64 + (utc.timestamp_subsec_nanos() as u64)
}

fn run(start: Instant, logger: &slog::Logger) -> Result<usize, String> {
    let opt = Opt::from_args();

    let mut n = 0;

    match opt {
        Opt::Range { trades_csv, output_path, start, end } => {
            let logger = logger.new(o!("cmd" => "range"));

            info!(logger, "beginning range cmd";
                "trades_csv" => %trades_csv.display(),
                "output_path" => %output_path.display(),
                "start" => %start,
                "end" => %end,
            );

            if ! trades_csv.exists() { return Err(format!("--trades-csv path does not exist: {}", trades_csv.display())) }

            info!(logger, "opening trades_csv file");

            let rdr = fs::File::open(&trades_csv)
                .map_err(|e| format!("opening trades csv file failed: {} (tried to open {})", e, trades_csv.display()))?;

            let rdr = io::BufReader::new(rdr);

            let mut rdr = csv::Reader::from_reader(rdr);

            let wtr = fs::File::create(&output_path)
                .map_err(|e| format!("opening output file failed: {} (tried to open {} for writing)", e, output_path.display()))?;

            let wtr = io::BufWriter::new(wtr);

            let mut wtr = csv::Writer::from_writer(wtr);

            let headers: csv::ByteRecord = rdr.byte_headers().map_err(|e| format!("failed to parse CSV headers: {}", e))?.clone();
            let mut row = csv::ByteRecord::new();

            let start_nanos = nanos(start);
            let end_nanos = nanos(end);

        }
    }

    Ok(n)
}

fn main() {
    let start = Instant::now();

    let decorator = slog_term::TermDecorator::new().stdout().force_color().build();
    let drain = slog_term::FullFormat::new(decorator).use_utc_timestamp().build().fuse();
    let drain = slog_async::Async::new(drain).chan_size(1024 * 64).thread_name("recv".into()).build().fuse();
    let logger = slog::Logger::root(drain, o!("version" => structopt::clap::crate_version!()));

    match run(start, &logger) {
        Ok(n) => {
            let took = Instant::now() - start;
            info!(logger, "finished in {:?}", took;
                "n rows" => %n.thousands_sep(),
                "rows/sec" => &((per_sec(n, took) * 100.0).round() / 10.0).thousands_sep(),
            );
        }

        Err(e) => {
            crit!(logger, "run failed: {:?}", e);
            eprintln!("\n\nError: {}", e);
            std::thread::sleep(Duration::from_millis(100));
            std::process::exit(1);
        }
    }

}