use std::num::{NonZeroU64, NonZeroU8, NonZeroI32}; use std::mem::size_of; use std::convert::{TryFrom, TryInto}; use serde::{Serialize, Deserialize, Deserializer}; use markets::crypto::{Exchange, Currency, Ticker, Side}; pub const EXCH_OFFSET : usize = 0; pub const BASE_OFFSET : usize = 1; pub const QUOTE_OFFSET : usize = 2; pub const SIDE_OFFSET : usize = 3; pub const SERVER_TIME_OFFSET : usize = 4; pub const TIME_OFFSET : usize = 8; pub const PRICE_OFFSET : usize = 16; pub const AMOUNT_OFFSET : usize = 24; pub const SERIALIZED_SIZE : usize = 32; /// `server_time` is stored in milliseconds, while `time` is nanoseconds. /// this is what you need to multiply the stored `server_time` data by to /// get it back to nanoseconds. pub const SERVER_TIME_DOWNSCALE_FACTOR: u64 = 1_000_000; /// Represents the serialized form of a trades row /// /// ```console,ignore /// 1 2 3 /// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 /// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ /// |e|b|q|s| srvtm | time: u64 | price: f64 | amount: f64 | /// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ /// | | | | | /// | | | | | /// | | | | | /// | | | | -> server_time: Option - 0=None, other=nano offset from `time` /// | | | | /// | | | -> side: Option - 0=None, 1=Bid, 2=Ask /// | | | /// | | -> quote: Currency - see markets::crypto for u8 <-> currency codes /// | | /// | -> base: Currency - see markets::crypto for u8 <-> currency codes /// | /// -> exch: Exchange - see markets::crypto for u8 <-> exchange codes /// /// ``` /// #[derive(Debug, Clone)] pub struct PackedTrade { pub exch: u8, pub base: u8, pub quote: u8, /// 0=None pub side: u8, /// relative offset from `time`; 0=None pub server_time: i32, pub time: u64, pub price: f64, pub amount: f64, } #[derive(Deserialize, Serialize, Debug, Clone)] pub struct CsvTrade { pub time: u64, pub exch: Exchange, pub ticker: Ticker, pub price: f64, pub amount: f64, #[serde(deserialize_with = "deserialize_csv_side")] pub side: Option, #[serde(deserialize_with = "deserialize_csv_server_time")] pub server_time: Option, } #[derive(Debug, Clone)] pub struct ParseError(Box); /// Pull out individual fields on demand from the serialized bytes of a PackedTrade #[repr(align(32))] pub struct PackedTradeData<'a>(&'a [u8]); #[derive(Deserialize, Serialize, Debug, Clone)] pub struct Serde32BytesTrade { pub time: u64, #[serde(with = "try_from_u8")] pub exch: Exchange, #[serde(with = "try_from_u8")] pub ticker: Ticker, pub price: f64, pub amount: f64, pub side: Option, pub server_time: Option, } pub fn server_time_to_delta(time: u64, server_time: u64) -> i32 { let ms = ( (server_time / SERVER_TIME_DOWNSCALE_FACTOR) as i64 - (time / SERVER_TIME_DOWNSCALE_FACTOR) as i64 ) as i32; match ms { // if the two values are either identical, or so close that the difference // is washed out when we downscale, return i32::MIN as a sentinel indicating // time == server_time // 0 => std::i32::MIN, other => other } } /// Convert a `server_time` delta back to its unix nanosecond timestamp form. /// /// Note: while the `server_time` delta is stored as a signed integer, to be able to express a /// delta in both directions relative to `time`, we can't just add a negative `i64` to a /// `u64`, it doesn't work like that. this match either subtracts the absolute value of a /// negative delta, or adds a positive delta, to get around this conundrum. pub fn delta_to_server_time(time: u64, delta: i32) -> Option { const MIN_VALID: i32 = std::i32::MIN + 1; match delta { 0 => None, // -1 is another sentinel indicating that time == server_time std::i32::MIN => Some(time), x @ MIN_VALID .. 0 => Some(time - (x.abs() as u64 * SERVER_TIME_DOWNSCALE_FACTOR)), x @ 1 ..= std::i32::MAX => Some(time + (x as u64 * SERVER_TIME_DOWNSCALE_FACTOR)), } } pub fn serialize<'a, 'b>(buf: &'a mut [u8], trade: &'b CsvTrade) { assert_eq!(buf.len(), SERIALIZED_SIZE); buf[EXCH_OFFSET] = u8::from(trade.exch); buf[BASE_OFFSET] = u8::from(trade.ticker.base); buf[QUOTE_OFFSET] = u8::from(trade.ticker.quote); match trade.side { Some(side) => { buf[SIDE_OFFSET] = u8::from(side); } None => { buf[SIDE_OFFSET] = 0; } } match trade.server_time { Some(st) => { let delta: i32 = server_time_to_delta(trade.time, st); (&mut buf[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).copy_from_slice(&delta.to_le_bytes()[..]); } None => { (&mut buf[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).copy_from_slice(&0i32.to_le_bytes()[..]); } } (&mut buf[TIME_OFFSET..(TIME_OFFSET + 8)]).copy_from_slice(&trade.time.to_le_bytes()[..]); (&mut buf[PRICE_OFFSET..(PRICE_OFFSET + 8)]).copy_from_slice(&trade.price.to_le_bytes()[..]); (&mut buf[AMOUNT_OFFSET..(AMOUNT_OFFSET + 8)]).copy_from_slice(&trade.amount.to_le_bytes()[..]); } impl<'a> PackedTradeData<'a> { pub fn new(buf: &'a [u8]) -> Self { assert_eq!(buf.len(), SERIALIZED_SIZE); Self(buf) } #[inline] pub fn exch(&self) -> Result { Exchange::try_from(self.0[EXCH_OFFSET]) } #[inline] pub fn base(&self) -> Result { Currency::try_from(self.0[BASE_OFFSET]) } #[inline] pub fn quote(&self) -> Result { Currency::try_from(self.0[QUOTE_OFFSET]) } #[inline] pub fn side(&self) -> Result, markets::crypto::Error> { match self.0[SIDE_OFFSET] { 0 => Ok(None), other => Ok(Some(Side::try_from(other)?)), } } #[inline] pub fn ticker(&self) -> Ticker { Ticker { base: self.base().unwrap(), quote: self.quote().unwrap(), } } #[inline] pub fn meta_i32(&self) -> i32 { i32::from_le_bytes((&self.0[..4]).try_into().unwrap()) } #[inline] pub fn time(&self) -> u64 { u64::from_le_bytes( (&self.0[TIME_OFFSET..(TIME_OFFSET + 8)]).try_into().unwrap() ) } #[inline] pub fn price(&self) -> f64 { f64::from_le_bytes( (&self.0[PRICE_OFFSET..(PRICE_OFFSET + 8)]).try_into().unwrap() ) } #[inline] pub fn amount(&self) -> f64 { f64::from_le_bytes( (&self.0[AMOUNT_OFFSET..(AMOUNT_OFFSET + 8)]).try_into().unwrap() ) } #[inline] pub fn server_time(&self) -> Option { let delta = i32::from_le_bytes( (&self.0[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).try_into().unwrap() ); delta_to_server_time(self.time(), delta) } } pub fn deserialize_csv_side<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de> { let s: &str = Deserialize::deserialize(deserializer)?; match s { "bid" => Ok(Some(Side::Bid)), "ask" => Ok(Some(Side::Ask)), _ => Ok(None) } } pub fn deserialize_csv_server_time<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de> { let st: u64 = Deserialize::deserialize(deserializer)?; match st { 0 => Ok(None), other => Ok(Some(other)) } } mod try_from_u8 { use std::convert::TryFrom; use std::fmt; use std::marker::PhantomData; use serde::{Serializer, Deserializer}; use serde::de::Visitor; use serde::ser::Error as SerError; struct V(PhantomData); impl<'de, T> Visitor<'de> for V where T: TryFrom { type Value = T; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("an integer code between 1-255") } fn visit_u8(self, v: u8) -> Result where E: serde::de::Error, { match T::try_from(v) { Ok(v) => Ok(v), Err(_) => { Err(serde::de::Error::custom("Invalid code")) } } } fn visit_u64(self, v: u64) -> Result where E: serde::de::Error, { if v > 255 { return Err(serde::de::Error::custom("Value greater than 255")) } match T::try_from(v as u8) { Ok(v) => Ok(v), Err(_) => { Err(serde::de::Error::custom("Invalid code")) } } } } pub fn deserialize<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, T: TryFrom { deserializer.deserialize_u8(V(PhantomData)) } pub fn serialize(item: &T, serializer: S) -> Result where S: Serializer, T: Copy, u8: From { match u8::from(*item) { 0 => Err(S::Error::custom("not implemented: no code for variant or value")), x => serializer.serialize_u8(x) } } } #[allow(unused)] #[cfg(test)] mod tests { use super::*; use std::io::{self, prelude::*}; use markets::{e, t, c}; use approx::assert_relative_eq; const CSV: &str = "time,amount,exch,price,server_time,side,ticker\n\ 1561939200002479372,1.4894,bnce,292.7,1561939199919000064,,eth_usd\n\ 1561939200011035644,0.0833333283662796,btfx,10809.0,1561939199927000064,bid,btc_usd\n\ 1561939200011055712,0.8333191871643066,btfx,10809.0,1561939199927000064,bid,btc_usd\n\ 1561939200019037617,0.083096,bnce,10854.1,1561939199935000064,,btc_usd\n\ 1561939200026450471,0.125,okex,123.21,1561939200026450432,ask,ltc_usd\n\ 1561939200027716312,0.704054,okex,123.21,1561939200027716352,ask,ltc_usd\n\ 1561939200028633907,0.11,okex,123.22,1561939200028633856,bid,ltc_usd\n\ 1561939200029908535,1.438978,okex,123.22,1561939200029908480,ask,ltc_usd\n\ 1561939200030393495,0.257589,okex,123.22,1561939200030393600,bid,ltc_usd" ; #[test] fn parse_csv_sample_with_csv_trade() { let csv: Vec = CSV.as_bytes().to_vec(); let mut rdr = csv::Reader::from_reader(io::Cursor::new(csv)); let mut rows = Vec::new(); let headers = rdr.byte_headers().unwrap().clone(); let mut row = csv::ByteRecord::new(); while rdr.read_byte_record(&mut row).unwrap() { let trade: CsvTrade = row.deserialize(Some(&headers)).unwrap(); rows.push(trade); } assert_eq!(rows[0].time, 1561939200002479372); assert_eq!(rows[1].exch, e!(btfx)); let mut buf = vec![0u8; 32]; for (i, trade) in rows.iter().enumerate() { assert!(trade.server_time.is_some()); let st = trade.server_time.unwrap(); let delta = server_time_to_delta(trade.time, st); dbg!(i, trade, trade.time, st, trade.time as i64 - st as i64, delta, (trade.time / SERVER_TIME_DOWNSCALE_FACTOR) as i64 - (st / SERVER_TIME_DOWNSCALE_FACTOR) as i64, ); assert!(delta != 0); let rt: u64 = delta_to_server_time(trade.time, delta).unwrap(); let abs_diff = (rt as i64 - st as i64).abs(); let max_allowable_diff = SERVER_TIME_DOWNSCALE_FACTOR; // * 2; dbg!(rt, abs_diff, max_allowable_diff); assert!(abs_diff < max_allowable_diff as i64); serialize(&mut buf[..], &trade); { let packed = PackedTradeData(&buf[..]); assert_eq!(packed.time(), trade.time); assert_eq!(packed.exch().unwrap(), trade.exch); assert_eq!(packed.base().unwrap(), trade.ticker.base); assert_eq!(packed.quote().unwrap(), trade.ticker.quote); assert_eq!(packed.side().unwrap(), trade.side); assert_relative_eq!(packed.price(), trade.price); assert_relative_eq!(packed.amount(), trade.amount); } } } #[test] fn verify_packed_trade_is_32_bytes() { assert_eq!(size_of::(), 32); } #[test] fn check_bincode_serialized_size() { let trade = Serde32BytesTrade { time: 1586996977191449698, exch: e!(bmex), ticker: t!(btc-usd), price: 1.234, amount: 4.567, side: None, server_time: NonZeroI32::new(1_000_000), }; assert_eq!(size_of::(), 32); assert_eq!(bincode::serialized_size(&trade).unwrap(), 32); } #[test] fn example_of_36_byte_trades_struct_without_the_offset_i32() { #[repr(packed)] pub struct Trade36 { pub exch: Exchange, pub ticker: Ticker, pub side: Option, pub time: u64, pub price: f64, pub amount: f64, pub server_time: Option, } assert_eq!(size_of::(), 36); } }