data-pipelines/src/encoding.rs

use std::num::{NonZeroU64, NonZeroU8, NonZeroI32};
use std::mem::size_of;
use std::convert::{TryFrom, TryInto};
use serde::{Serialize, Deserialize, Deserializer};
use markets::crypto::{Exchange, Currency, Ticker, Side};

pub const EXCH_OFFSET           : usize = 0;
pub const BASE_OFFSET           : usize = 1;
pub const QUOTE_OFFSET          : usize = 2;
pub const SIDE_OFFSET           : usize = 3;
pub const SERVER_TIME_OFFSET    : usize = 4;
pub const TIME_OFFSET           : usize = 8;
pub const PRICE_OFFSET          : usize = 16;
pub const AMOUNT_OFFSET         : usize = 24;
pub const SERIALIZED_SIZE       : usize = 32;

/// `server_time` is stored in milliseconds, while `time` is nanoseconds.
/// this is what you need to multiply the stored `server_time` data by to
/// get it back to nanoseconds.
pub const SERVER_TIME_DOWNSCALE_FACTOR: u64 = 1_000_000;



/// Represents the serialized form of a trades row
///
/// ```console,ignore
///                      1                   2                   3
///  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/// |e|b|q|s| srvtm | time: u64     | price: f64    | amount: f64   |
/// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
///  | | | | |
///  | | | | |
///  | | | | |
///  | | | | -> server_time: Option<i32> - 0=None, other=nano offset from `time`
///  | | | |
///  | | | -> side: Option<Side> - 0=None, 1=Bid, 2=Ask
///  | | |  
///  | | -> quote: Currency - see markets::crypto for u8 <-> currency codes
///  | |    
///  | -> base: Currency - see markets::crypto for u8 <-> currency codes
///  |     
///  -> exch: Exchange - see markets::crypto for u8 <-> exchange codes
///
///  ```
///
#[derive(Debug, Clone)]
pub struct PackedTrade {
    pub exch: u8,
    pub base: u8,
    pub quote: u8,

    /// 0=None
    pub side: u8,

    /// relative offset from `time`; 0=None
    pub server_time: i32,

    pub time: u64,
    pub price: f64,
    pub amount: f64,
}

#[derive(Deserialize, Serialize, Debug, Clone)]

pub struct CsvTrade {
    pub time: u64,
    pub exch: Exchange,
    pub ticker: Ticker,
    pub price: f64,
    pub amount: f64,
    #[serde(deserialize_with = "deserialize_csv_side")]
    pub side: Option<Side>,
    #[serde(deserialize_with = "deserialize_csv_server_time")]
    pub server_time: Option<u64>,
}

#[derive(Debug, Clone)]
pub struct ParseError(Box<String>);

/// Pull out individual fields on demand from the serialized bytes of a PackedTrade
#[repr(align(32))]
pub struct PackedTradeData<'a>(&'a [u8]);

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct Serde32BytesTrade {
    pub time: u64,
    #[serde(with = "try_from_u8")]
    pub exch: Exchange,
    #[serde(with = "try_from_u8")]
    pub ticker: Ticker,
    pub price: f64,
    pub amount: f64,
    pub side: Option<Side>,
    pub server_time: Option<NonZeroI32>,
}

pub fn server_time_to_delta(time: u64, server_time: u64) -> i32 {
    let ms = (
        (server_time / SERVER_TIME_DOWNSCALE_FACTOR) as i64
        - (time / SERVER_TIME_DOWNSCALE_FACTOR) as i64
    ) as i32;

    match ms {
        // if the two values are either identical, or so close that the difference
        // is washed out when we downscale, return i32::MIN as a sentinel indicating
        // time == server_time
        //
        0 => std::i32::MIN,

        other => other
    }
}

/// Convert a `server_time` delta back to its unix nanosecond timestamp form.
///
/// Note: while the `server_time` delta is stored as a signed integer, to be able to express a
/// delta in both directions relative to `time`, we can't just add a negative `i64` to a
/// `u64`, it doesn't work like that. this match either subtracts the absolute value of a
/// negative delta, or adds a positive delta, to get around this conundrum.
pub fn delta_to_server_time(time: u64, delta: i32) -> Option<u64> {
    const MIN_VALID: i32 = std::i32::MIN + 1;

    match delta {
        0 => None,

        // -1 is another sentinel indicating that time == server_time 
        std::i32::MIN => Some(time),

        x @ MIN_VALID .. 0  => Some(time - (x.abs() as u64 * SERVER_TIME_DOWNSCALE_FACTOR)),

        x @ 1 ..= std::i32::MAX => Some(time + (x as u64 * SERVER_TIME_DOWNSCALE_FACTOR)),
    }
}


pub fn serialize<'a, 'b>(buf: &'a mut [u8], trade: &'b CsvTrade) {
    assert_eq!(buf.len(), SERIALIZED_SIZE);

    buf[EXCH_OFFSET] = u8::from(trade.exch);
    buf[BASE_OFFSET] = u8::from(trade.ticker.base);
    buf[QUOTE_OFFSET] = u8::from(trade.ticker.quote);

    match trade.side {
        Some(side) => {
            buf[SIDE_OFFSET] = u8::from(side);
        }

        None => {
            buf[SIDE_OFFSET] = 0;
        }
    }

    match trade.server_time {
        Some(st) => {
            let delta: i32 = server_time_to_delta(trade.time, st);
            (&mut buf[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).copy_from_slice(&delta.to_le_bytes()[..]);
        }

        None => {
            (&mut buf[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).copy_from_slice(&0i32.to_le_bytes()[..]);
        }
    }
    
    (&mut buf[TIME_OFFSET..(TIME_OFFSET + 8)]).copy_from_slice(&trade.time.to_le_bytes()[..]);
    (&mut buf[PRICE_OFFSET..(PRICE_OFFSET + 8)]).copy_from_slice(&trade.price.to_le_bytes()[..]);
    (&mut buf[AMOUNT_OFFSET..(AMOUNT_OFFSET + 8)]).copy_from_slice(&trade.amount.to_le_bytes()[..]);
}


impl<'a> PackedTradeData<'a> {
    pub fn new(buf: &'a [u8]) -> Self {
        assert_eq!(buf.len(), SERIALIZED_SIZE);
        Self(buf)
    }

    #[inline]
    pub fn exch(&self) -> Result<Exchange, markets::crypto::Error> {
        Exchange::try_from(self.0[EXCH_OFFSET])
    }

    #[inline]
    pub fn base(&self) -> Result<Currency, markets::crypto::Error> {
        Currency::try_from(self.0[BASE_OFFSET])
    }
    #[inline]
    pub fn ticker(&self) -> Ticker {
        Ticker {
            base: self.base().unwrap(),
            quote: self.quote().unwrap(),
        }
    }

    #[inline]
    pub fn quote(&self) -> Result<Currency, markets::crypto::Error> {
        Currency::try_from(self.0[QUOTE_OFFSET])
    }

    #[inline]
    pub fn side(&self) -> Result<Option<Side>, markets::crypto::Error> {
        match self.0[SIDE_OFFSET] {
            0 => Ok(None),
            other => Ok(Some(Side::try_from(other)?)),
        }
    }

    #[inline]
    pub fn time(&self) -> u64 {
        u64::from_le_bytes(
            (&self.0[TIME_OFFSET..(TIME_OFFSET + 8)]).try_into().unwrap()
        )
    }

    #[inline]
    pub fn price(&self) -> f64 {
        f64::from_le_bytes(
            (&self.0[PRICE_OFFSET..(PRICE_OFFSET + 8)]).try_into().unwrap()
        )
    }

    #[inline]
    pub fn amount(&self) -> f64 {
        f64::from_le_bytes(
            (&self.0[AMOUNT_OFFSET..(AMOUNT_OFFSET + 8)]).try_into().unwrap()
        )
    }

    #[inline]
    pub fn server_time(&self) -> Option<u64> {
        let delta = i32::from_le_bytes(
            (&self.0[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).try_into().unwrap()
        );

        delta_to_server_time(self.time(), delta)
    }
}

pub fn deserialize_csv_side<'de, D>(deserializer: D) -> Result<Option<Side>, D::Error>
    where D: Deserializer<'de>
{
    let s: &str = Deserialize::deserialize(deserializer)?;
    match s {
        "bid" => Ok(Some(Side::Bid)),
        "ask" => Ok(Some(Side::Ask)),
        _ => Ok(None)
    }
}

pub fn deserialize_csv_server_time<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
    where D: Deserializer<'de>
{
    let st: u64 = Deserialize::deserialize(deserializer)?;
    match st {
        0 => Ok(None),
        other => Ok(Some(other))
    }
}

mod try_from_u8 {
    use std::convert::TryFrom;
    use std::fmt;
    use std::marker::PhantomData;
    use serde::{Serializer, Deserializer};
    use serde::de::Visitor;
    use serde::ser::Error as SerError;

    struct V<T>(PhantomData<T>);

    impl<'de, T> Visitor<'de> for V<T>
        where T: TryFrom<u8>
    {
        type Value = T;

        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
            formatter.write_str("an integer code between 1-255")
        }

        fn visit_u8<E>(self, v: u8) -> Result<Self::Value, E>
            where E: serde::de::Error,
        {
            match T::try_from(v) {
                Ok(v) => Ok(v),
                Err(_) => {
                    Err(serde::de::Error::custom("Invalid code"))
                }
            }
        }

        fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
            where E: serde::de::Error,
        {
            if v > 255 {
                return Err(serde::de::Error::custom("Value greater than 255"))
            }
            match T::try_from(v as u8) {
                Ok(v) => Ok(v),
                Err(_) => {
                    Err(serde::de::Error::custom("Invalid code"))
                }
            }
        }
    }


    pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
        where D: Deserializer<'de>,
              T: TryFrom<u8>
    {
        deserializer.deserialize_u8(V(PhantomData))
    }

    pub fn serialize<S, T>(item: &T, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer,
              T: Copy,
              u8: From<T>
    {
        match u8::from(*item) {
            0 => Err(S::Error::custom("not implemented: no code for variant or value")),
            x => serializer.serialize_u8(x)
        }
    }
}

#[allow(unused)]
#[cfg(test)]
mod tests {
    use super::*;
    use std::io::{self, prelude::*};
    use markets::{e, t, c};
    use approx::assert_relative_eq;

    const CSV: &str =
        "time,amount,exch,price,server_time,side,ticker\n\
         1561939200002479372,1.4894,bnce,292.7,1561939199919000064,,eth_usd\n\
         1561939200011035644,0.0833333283662796,btfx,10809.0,1561939199927000064,bid,btc_usd\n\
         1561939200011055712,0.8333191871643066,btfx,10809.0,1561939199927000064,bid,btc_usd\n\
         1561939200019037617,0.083096,bnce,10854.1,1561939199935000064,,btc_usd\n\
         1561939200026450471,0.125,okex,123.21,1561939200026450432,ask,ltc_usd\n\
         1561939200027716312,0.704054,okex,123.21,1561939200027716352,ask,ltc_usd\n\
         1561939200028633907,0.11,okex,123.22,1561939200028633856,bid,ltc_usd\n\
         1561939200029908535,1.438978,okex,123.22,1561939200029908480,ask,ltc_usd\n\
         1561939200030393495,0.257589,okex,123.22,1561939200030393600,bid,ltc_usd"
         ;

    #[test]
    fn parse_csv_sample_with_csv_trade() {
        let csv: Vec<u8> = CSV.as_bytes().to_vec();
        let mut rdr = csv::Reader::from_reader(io::Cursor::new(csv));
        let mut rows = Vec::new();
        let headers = rdr.byte_headers().unwrap().clone();
        let mut row = csv::ByteRecord::new();
        while rdr.read_byte_record(&mut row).unwrap() {
            let trade: CsvTrade = row.deserialize(Some(&headers)).unwrap();
            rows.push(trade);
        }

        assert_eq!(rows[0].time, 1561939200002479372);
        assert_eq!(rows[1].exch, e!(btfx));

        let mut buf = vec![0u8; 32];

        for (i, trade) in rows.iter().enumerate() {
            assert!(trade.server_time.is_some());
            let st = trade.server_time.unwrap();
            let delta = server_time_to_delta(trade.time, st);
            dbg!(i, trade, trade.time, st,
                trade.time as i64 - st as i64, delta,
                (trade.time / SERVER_TIME_DOWNSCALE_FACTOR) as i64 - (st / SERVER_TIME_DOWNSCALE_FACTOR) as i64,
            );
            assert!(delta != 0);
            let rt: u64 = delta_to_server_time(trade.time, delta).unwrap();
            let abs_diff = (rt as i64 - st as i64).abs();
            let max_allowable_diff = SERVER_TIME_DOWNSCALE_FACTOR; // * 2;
            dbg!(rt, abs_diff, max_allowable_diff);
            assert!(abs_diff < max_allowable_diff as i64);

            serialize(&mut buf[..], &trade);
            {
                let packed = PackedTradeData(&buf[..]);
                assert_eq!(packed.time(), trade.time);
                assert_eq!(packed.exch().unwrap(), trade.exch);
                assert_eq!(packed.base().unwrap(), trade.ticker.base);
                assert_eq!(packed.quote().unwrap(), trade.ticker.quote);
                assert_eq!(packed.side().unwrap(), trade.side);
                assert_relative_eq!(packed.price(), trade.price);
                assert_relative_eq!(packed.amount(), trade.amount);
            }
        }
    }

    #[test]
    fn verify_packed_trade_is_32_bytes() {
        assert_eq!(size_of::<PackedTrade>(), 32);
    }

    #[test]
    fn check_bincode_serialized_size() {
        let trade = Serde32BytesTrade {
            time: 1586996977191449698,
            exch: e!(bmex),
            ticker: t!(btc-usd),
            price: 1.234,
            amount: 4.567,
            side: None,
            server_time: NonZeroI32::new(1_000_000),
        };

        assert_eq!(size_of::<Serde32BytesTrade>(), 32);
        assert_eq!(bincode::serialized_size(&trade).unwrap(), 32);
    }

    #[test]
    fn example_of_36_byte_trades_struct_without_the_offset_i32() {
        #[repr(packed)]
        pub struct Trade36 {
            pub exch: Exchange,
            pub ticker: Ticker,
            pub side: Option<Side>,

            pub time: u64,
            pub price: f64,
            pub amount: f64,

            pub server_time: Option<NonZeroU64>,
        }

        assert_eq!(size_of::<Trade36>(), 36);
    }
}