|
- use std::num::{NonZeroU64, NonZeroU8, NonZeroI32};
- use std::mem::size_of;
- use std::convert::{TryFrom, TryInto};
- use serde::{Serialize, Deserialize, Deserializer};
- use markets::crypto::{Exchange, Currency, Ticker, Side};
-
- pub const EXCH_OFFSET : usize = 0;
- pub const BASE_OFFSET : usize = 1;
- pub const QUOTE_OFFSET : usize = 2;
- pub const SIDE_OFFSET : usize = 3;
- pub const SERVER_TIME_OFFSET : usize = 4;
- pub const TIME_OFFSET : usize = 8;
- pub const PRICE_OFFSET : usize = 16;
- pub const AMOUNT_OFFSET : usize = 24;
- pub const SERIALIZED_SIZE : usize = 32;
-
- /// `server_time` is stored in milliseconds, while `time` is nanoseconds.
- /// this is what you need to multiply the stored `server_time` data by to
- /// get it back to nanoseconds.
- pub const SERVER_TIME_DOWNSCALE_FACTOR: u64 = 1_000_000;
-
-
-
- /// Represents the serialized form of a trades row
- ///
- /// ```console,ignore
- /// 1 2 3
- /// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
- /// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- /// |e|b|q|s| srvtm | time: u64 | price: f64 | amount: f64 |
- /// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- /// | | | | |
- /// | | | | |
- /// | | | | |
- /// | | | | -> server_time: Option<i32> - 0=None, other=nano offset from `time`
- /// | | | |
- /// | | | -> side: Option<Side> - 0=None, 1=Bid, 2=Ask
- /// | | |
- /// | | -> quote: Currency - see markets::crypto for u8 <-> currency codes
- /// | |
- /// | -> base: Currency - see markets::crypto for u8 <-> currency codes
- /// |
- /// -> exch: Exchange - see markets::crypto for u8 <-> exchange codes
- ///
- /// ```
- ///
- #[derive(Debug, Clone)]
- pub struct PackedTrade {
- pub exch: u8,
- pub base: u8,
- pub quote: u8,
-
- /// 0=None
- pub side: u8,
-
- /// relative offset from `time`; 0=None
- pub server_time: i32,
-
- pub time: u64,
- pub price: f64,
- pub amount: f64,
- }
-
- #[derive(Deserialize, Serialize, Debug, Clone)]
-
- pub struct CsvTrade {
- pub time: u64,
- pub exch: Exchange,
- pub ticker: Ticker,
- pub price: f64,
- pub amount: f64,
- #[serde(deserialize_with = "deserialize_csv_side")]
- pub side: Option<Side>,
- #[serde(deserialize_with = "deserialize_csv_server_time")]
- pub server_time: Option<u64>,
- }
-
- #[derive(Debug, Clone)]
- pub struct ParseError(Box<String>);
-
- /// Pull out individual fields on demand from the serialized bytes of a PackedTrade
- #[repr(align(32))]
- pub struct PackedTradeData<'a>(&'a [u8]);
-
- #[derive(Deserialize, Serialize, Debug, Clone)]
- pub struct Serde32BytesTrade {
- pub time: u64,
- #[serde(with = "try_from_u8")]
- pub exch: Exchange,
- #[serde(with = "try_from_u8")]
- pub ticker: Ticker,
- pub price: f64,
- pub amount: f64,
- pub side: Option<Side>,
- pub server_time: Option<NonZeroI32>,
- }
-
- pub fn server_time_to_delta(time: u64, server_time: u64) -> i32 {
- let ms = (
- (server_time / SERVER_TIME_DOWNSCALE_FACTOR) as i64
- - (time / SERVER_TIME_DOWNSCALE_FACTOR) as i64
- ) as i32;
-
- match ms {
- // if the two values are either identical, or so close that the difference
- // is washed out when we downscale, return i32::MIN as a sentinel indicating
- // time == server_time
- //
- 0 => std::i32::MIN,
-
- other => other
- }
- }
-
- /// Convert a `server_time` delta back to its unix nanosecond timestamp form.
- ///
- /// Note: while the `server_time` delta is stored as a signed integer, to be able to express a
- /// delta in both directions relative to `time`, we can't just add a negative `i64` to a
- /// `u64`, it doesn't work like that. this match either subtracts the absolute value of a
- /// negative delta, or adds a positive delta, to get around this conundrum.
- pub fn delta_to_server_time(time: u64, delta: i32) -> Option<u64> {
- const MIN_VALID: i32 = std::i32::MIN + 1;
-
- match delta {
- 0 => None,
-
- // -1 is another sentinel indicating that time == server_time
- std::i32::MIN => Some(time),
-
- x @ MIN_VALID .. 0 => Some(time - (x.abs() as u64 * SERVER_TIME_DOWNSCALE_FACTOR)),
-
- x @ 1 ..= std::i32::MAX => Some(time + (x as u64 * SERVER_TIME_DOWNSCALE_FACTOR)),
- }
- }
-
-
- pub fn serialize<'a, 'b>(buf: &'a mut [u8], trade: &'b CsvTrade) {
- assert_eq!(buf.len(), SERIALIZED_SIZE);
-
- buf[EXCH_OFFSET] = u8::from(trade.exch);
- buf[BASE_OFFSET] = u8::from(trade.ticker.base);
- buf[QUOTE_OFFSET] = u8::from(trade.ticker.quote);
-
- match trade.side {
- Some(side) => {
- buf[SIDE_OFFSET] = u8::from(side);
- }
-
- None => {
- buf[SIDE_OFFSET] = 0;
- }
- }
-
- match trade.server_time {
- Some(st) => {
- let delta: i32 = server_time_to_delta(trade.time, st);
- (&mut buf[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).copy_from_slice(&delta.to_le_bytes()[..]);
- }
-
- None => {
- (&mut buf[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).copy_from_slice(&0i32.to_le_bytes()[..]);
- }
- }
-
- (&mut buf[TIME_OFFSET..(TIME_OFFSET + 8)]).copy_from_slice(&trade.time.to_le_bytes()[..]);
- (&mut buf[PRICE_OFFSET..(PRICE_OFFSET + 8)]).copy_from_slice(&trade.price.to_le_bytes()[..]);
- (&mut buf[AMOUNT_OFFSET..(AMOUNT_OFFSET + 8)]).copy_from_slice(&trade.amount.to_le_bytes()[..]);
- }
-
-
- impl<'a> PackedTradeData<'a> {
- pub fn new(buf: &'a [u8]) -> Self {
- assert_eq!(buf.len(), SERIALIZED_SIZE);
- Self(buf)
- }
-
- #[inline]
- pub fn exch(&self) -> Result<Exchange, markets::crypto::Error> {
- Exchange::try_from(self.0[EXCH_OFFSET])
- }
-
- #[inline]
- pub fn base(&self) -> Result<Currency, markets::crypto::Error> {
- Currency::try_from(self.0[BASE_OFFSET])
- }
-
- #[inline]
- pub fn quote(&self) -> Result<Currency, markets::crypto::Error> {
- Currency::try_from(self.0[QUOTE_OFFSET])
- }
-
- #[inline]
- pub fn side(&self) -> Result<Option<Side>, markets::crypto::Error> {
- match self.0[SIDE_OFFSET] {
- 0 => Ok(None),
- other => Ok(Some(Side::try_from(other)?)),
- }
- }
-
- #[inline]
- pub fn ticker(&self) -> Ticker {
- Ticker {
- base: self.base().unwrap(),
- quote: self.quote().unwrap(),
- }
- }
-
- #[inline]
- pub fn meta_i32(&self) -> i32 {
- i32::from_le_bytes((&self.0[..4]).try_into().unwrap())
- }
-
-
- #[inline]
- pub fn time(&self) -> u64 {
- u64::from_le_bytes(
- (&self.0[TIME_OFFSET..(TIME_OFFSET + 8)]).try_into().unwrap()
- )
- }
-
- #[inline]
- pub fn price(&self) -> f64 {
- f64::from_le_bytes(
- (&self.0[PRICE_OFFSET..(PRICE_OFFSET + 8)]).try_into().unwrap()
- )
- }
-
- #[inline]
- pub fn amount(&self) -> f64 {
- f64::from_le_bytes(
- (&self.0[AMOUNT_OFFSET..(AMOUNT_OFFSET + 8)]).try_into().unwrap()
- )
- }
-
- #[inline]
- pub fn server_time(&self) -> Option<u64> {
- let delta = i32::from_le_bytes(
- (&self.0[SERVER_TIME_OFFSET..(SERVER_TIME_OFFSET + 4)]).try_into().unwrap()
- );
-
- delta_to_server_time(self.time(), delta)
- }
- }
-
- pub fn deserialize_csv_side<'de, D>(deserializer: D) -> Result<Option<Side>, D::Error>
- where D: Deserializer<'de>
- {
- let s: &str = Deserialize::deserialize(deserializer)?;
- match s {
- "bid" => Ok(Some(Side::Bid)),
- "ask" => Ok(Some(Side::Ask)),
- _ => Ok(None)
- }
- }
-
- pub fn deserialize_csv_server_time<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
- where D: Deserializer<'de>
- {
- let st: u64 = Deserialize::deserialize(deserializer)?;
- match st {
- 0 => Ok(None),
- other => Ok(Some(other))
- }
- }
-
- mod try_from_u8 {
- use std::convert::TryFrom;
- use std::fmt;
- use std::marker::PhantomData;
- use serde::{Serializer, Deserializer};
- use serde::de::Visitor;
- use serde::ser::Error as SerError;
-
- struct V<T>(PhantomData<T>);
-
- impl<'de, T> Visitor<'de> for V<T>
- where T: TryFrom<u8>
- {
- type Value = T;
-
- fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
- formatter.write_str("an integer code between 1-255")
- }
-
- fn visit_u8<E>(self, v: u8) -> Result<Self::Value, E>
- where E: serde::de::Error,
- {
- match T::try_from(v) {
- Ok(v) => Ok(v),
- Err(_) => {
- Err(serde::de::Error::custom("Invalid code"))
- }
- }
- }
-
- fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
- where E: serde::de::Error,
- {
- if v > 255 {
- return Err(serde::de::Error::custom("Value greater than 255"))
- }
- match T::try_from(v as u8) {
- Ok(v) => Ok(v),
- Err(_) => {
- Err(serde::de::Error::custom("Invalid code"))
- }
- }
- }
- }
-
-
- pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
- where D: Deserializer<'de>,
- T: TryFrom<u8>
- {
- deserializer.deserialize_u8(V(PhantomData))
- }
-
- pub fn serialize<S, T>(item: &T, serializer: S) -> Result<S::Ok, S::Error>
- where S: Serializer,
- T: Copy,
- u8: From<T>
- {
- match u8::from(*item) {
- 0 => Err(S::Error::custom("not implemented: no code for variant or value")),
- x => serializer.serialize_u8(x)
- }
- }
- }
-
- #[allow(unused)]
- #[cfg(test)]
- mod tests {
- use super::*;
- use std::io::{self, prelude::*};
- use markets::{e, t, c};
- use approx::assert_relative_eq;
-
- const CSV: &str =
- "time,amount,exch,price,server_time,side,ticker\n\
- 1561939200002479372,1.4894,bnce,292.7,1561939199919000064,,eth_usd\n\
- 1561939200011035644,0.0833333283662796,btfx,10809.0,1561939199927000064,bid,btc_usd\n\
- 1561939200011055712,0.8333191871643066,btfx,10809.0,1561939199927000064,bid,btc_usd\n\
- 1561939200019037617,0.083096,bnce,10854.1,1561939199935000064,,btc_usd\n\
- 1561939200026450471,0.125,okex,123.21,1561939200026450432,ask,ltc_usd\n\
- 1561939200027716312,0.704054,okex,123.21,1561939200027716352,ask,ltc_usd\n\
- 1561939200028633907,0.11,okex,123.22,1561939200028633856,bid,ltc_usd\n\
- 1561939200029908535,1.438978,okex,123.22,1561939200029908480,ask,ltc_usd\n\
- 1561939200030393495,0.257589,okex,123.22,1561939200030393600,bid,ltc_usd"
- ;
-
- #[test]
- fn parse_csv_sample_with_csv_trade() {
- let csv: Vec<u8> = CSV.as_bytes().to_vec();
- let mut rdr = csv::Reader::from_reader(io::Cursor::new(csv));
- let mut rows = Vec::new();
- let headers = rdr.byte_headers().unwrap().clone();
- let mut row = csv::ByteRecord::new();
- while rdr.read_byte_record(&mut row).unwrap() {
- let trade: CsvTrade = row.deserialize(Some(&headers)).unwrap();
- rows.push(trade);
- }
-
- assert_eq!(rows[0].time, 1561939200002479372);
- assert_eq!(rows[1].exch, e!(btfx));
-
- let mut buf = vec![0u8; 32];
-
- for (i, trade) in rows.iter().enumerate() {
- assert!(trade.server_time.is_some());
- let st = trade.server_time.unwrap();
- let delta = server_time_to_delta(trade.time, st);
- dbg!(i, trade, trade.time, st,
- trade.time as i64 - st as i64, delta,
- (trade.time / SERVER_TIME_DOWNSCALE_FACTOR) as i64 - (st / SERVER_TIME_DOWNSCALE_FACTOR) as i64,
- );
- assert!(delta != 0);
- let rt: u64 = delta_to_server_time(trade.time, delta).unwrap();
- let abs_diff = (rt as i64 - st as i64).abs();
- let max_allowable_diff = SERVER_TIME_DOWNSCALE_FACTOR; // * 2;
- dbg!(rt, abs_diff, max_allowable_diff);
- assert!(abs_diff < max_allowable_diff as i64);
-
- serialize(&mut buf[..], &trade);
- {
- let packed = PackedTradeData(&buf[..]);
- assert_eq!(packed.time(), trade.time);
- assert_eq!(packed.exch().unwrap(), trade.exch);
- assert_eq!(packed.base().unwrap(), trade.ticker.base);
- assert_eq!(packed.quote().unwrap(), trade.ticker.quote);
- assert_eq!(packed.side().unwrap(), trade.side);
- assert_relative_eq!(packed.price(), trade.price);
- assert_relative_eq!(packed.amount(), trade.amount);
- }
- }
- }
-
- #[test]
- fn verify_packed_trade_is_32_bytes() {
- assert_eq!(size_of::<PackedTrade>(), 32);
- }
-
- #[test]
- fn check_bincode_serialized_size() {
- let trade = Serde32BytesTrade {
- time: 1586996977191449698,
- exch: e!(bmex),
- ticker: t!(btc-usd),
- price: 1.234,
- amount: 4.567,
- side: None,
- server_time: NonZeroI32::new(1_000_000),
- };
-
- assert_eq!(size_of::<Serde32BytesTrade>(), 32);
- assert_eq!(bincode::serialized_size(&trade).unwrap(), 32);
- }
-
- #[test]
- fn example_of_36_byte_trades_struct_without_the_offset_i32() {
- #[repr(packed)]
- pub struct Trade36 {
- pub exch: Exchange,
- pub ticker: Ticker,
- pub side: Option<Side>,
-
- pub time: u64,
- pub price: f64,
- pub amount: f64,
-
- pub server_time: Option<NonZeroU64>,
- }
-
- assert_eq!(size_of::<Trade36>(), 36);
- }
- }
|