connectorx/transports/
clickhouse_arrow.rs

1//! Transport from ClickHouse Source to Arrow Destination.
2
3use crate::{
4    destinations::arrow::{typesystem::ArrowTypeSystem, ArrowDestination, ArrowDestinationError},
5    impl_transport,
6    sources::clickhouse::{ClickHouseSource, ClickHouseSourceError, ClickHouseTypeSystem},
7    typesystem::TypeConversion,
8};
9use chrono::{DateTime, NaiveDate, NaiveTime, Utc};
10use rust_decimal::Decimal;
11use std::net::IpAddr;
12use thiserror::Error;
13use uuid::Uuid;
14
15#[derive(Error, Debug)]
16pub enum ClickHouseArrowTransportError {
17    #[error(transparent)]
18    Source(#[from] ClickHouseSourceError),
19
20    #[error(transparent)]
21    Destination(#[from] ArrowDestinationError),
22
23    #[error(transparent)]
24    ConnectorX(#[from] crate::errors::ConnectorXError),
25}
26
27/// Convert ClickHouse data types to Arrow data types.
28pub struct ClickHouseArrowTransport;
29
30impl_transport!(
31    name = ClickHouseArrowTransport,
32    error = ClickHouseArrowTransportError,
33    systems = ClickHouseTypeSystem => ArrowTypeSystem,
34    route = ClickHouseSource => ArrowDestination,
35    mappings = {
36        { Int8[i8]                   => Int16[i16]                              | conversion auto }
37        { Int16[i16]                 => Int16[i16]                              | conversion auto }
38        { Int32[i32]                 => Int32[i32]                              | conversion auto }
39        { Int64[i64]                 => Int64[i64]                              | conversion auto }
40
41        { UInt8[u8]                  => UInt16[u16]                             | conversion auto }
42        { UInt16[u16]                => UInt16[u16]                             | conversion auto }
43        { UInt32[u32]                => UInt32[u32]                             | conversion auto }
44        { UInt64[u64]                => UInt64[u64]                             | conversion auto }
45
46        { Float32[f32]               => Float32[f32]                            | conversion auto }
47        { Float64[f64]               => Float64[f64]                            | conversion auto }
48
49        { Decimal[Decimal]           => Decimal[Decimal]                        | conversion auto }
50
51        { String[String]             => LargeUtf8[String]                       | conversion auto }
52        { FixedString[Vec<u8>]       => LargeBinary[Vec<u8>]                    | conversion auto }
53
54        { Enum8[String]              => LargeUtf8[String]                       | conversion none }
55        { Enum16[String]             => LargeUtf8[String]                       | conversion none }
56
57        { Date[NaiveDate]            => Date32[NaiveDate]                       | conversion auto }
58        { Date32[NaiveDate]          => Date32[NaiveDate]                       | conversion none }
59        { DateTime[DateTime<Utc>]    => DateTimeTz[DateTime<Utc>]               | conversion auto }
60        { DateTime64[DateTime<Utc>]  => DateTimeTz[DateTime<Utc>]               | conversion none }
61        { Time[NaiveTime]            => Time64[NaiveTime]                       | conversion auto }
62        { Time64[NaiveTime]          => Time64[NaiveTime]                       | conversion none }
63
64        { UUID[Uuid]                 => LargeUtf8[String]                       | conversion option }
65        { IPv4[IpAddr]               => LargeUtf8[String]                       | conversion option }
66        { IPv6[IpAddr]               => LargeUtf8[String]                       | conversion none }
67        { Bool[bool]                 => Boolean[bool]                           | conversion auto }
68
69        { ArrayBool[Vec<Option<bool>>]         => BoolArray[Vec<Option<bool>>]           | conversion auto }
70        { ArrayString[Vec<Option<String>>]     => Utf8Array[Vec<Option<String>>]         | conversion auto }
71        { ArrayInt8[Vec<Option<i8>>]           => Int16Array[Vec<Option<i16>>]           | conversion none }
72        { ArrayInt16[Vec<Option<i16>>]         => Int16Array[Vec<Option<i16>>]           | conversion auto }
73        { ArrayInt32[Vec<Option<i32>>]         => Int32Array[Vec<Option<i32>>]           | conversion auto }
74        { ArrayInt64[Vec<Option<i64>>]         => Int64Array[Vec<Option<i64>>]           | conversion auto }
75        { ArrayUInt8[Vec<Option<u8>>]          => UInt16Array[Vec<Option<u16>>]          | conversion none }
76        { ArrayUInt16[Vec<Option<u16>>]        => UInt16Array[Vec<Option<u16>>]          | conversion auto }
77        { ArrayUInt32[Vec<Option<u32>>]        => UInt32Array[Vec<Option<u32>>]          | conversion auto }
78        { ArrayUInt64[Vec<Option<u64>>]        => UInt64Array[Vec<Option<u64>>]          | conversion auto }
79        { ArrayFloat32[Vec<Option<f32>>]       => Float32Array[Vec<Option<f32>>]         | conversion auto }
80        { ArrayFloat64[Vec<Option<f64>>]       => Float64Array[Vec<Option<f64>>]         | conversion auto }
81        { ArrayDecimal[Vec<Option<Decimal>>]   => DecimalArray[Vec<Option<Decimal>>]     | conversion auto }
82    }
83);
84
85impl TypeConversion<Uuid, String> for ClickHouseArrowTransport {
86    fn convert(val: Uuid) -> String {
87        val.to_string()
88    }
89}
90
91impl TypeConversion<IpAddr, String> for ClickHouseArrowTransport {
92    fn convert(val: IpAddr) -> String {
93        val.to_string()
94    }
95}
96
97impl TypeConversion<Vec<Option<i8>>, Vec<Option<i16>>> for ClickHouseArrowTransport {
98    fn convert(val: Vec<Option<i8>>) -> Vec<Option<i16>> {
99        val.into_iter().map(|opt| opt.map(|v| v as i16)).collect()
100    }
101}
102
103impl TypeConversion<Vec<Option<u8>>, Vec<Option<u16>>> for ClickHouseArrowTransport {
104    fn convert(val: Vec<Option<u8>>) -> Vec<Option<u16>> {
105        val.into_iter().map(|opt| opt.map(|v| v as u16)).collect()
106    }
107}
108
109impl TypeConversion<Option<Vec<Option<i8>>>, Option<Vec<Option<i16>>>>
110    for ClickHouseArrowTransport
111{
112    fn convert(val: Option<Vec<Option<i8>>>) -> Option<Vec<Option<i16>>> {
113        val.map(Self::convert)
114    }
115}
116
117impl TypeConversion<Option<Vec<Option<u8>>>, Option<Vec<Option<u16>>>>
118    for ClickHouseArrowTransport
119{
120    fn convert(val: Option<Vec<Option<u8>>>) -> Option<Vec<Option<u16>>> {
121        val.map(Self::convert)
122    }
123}
connectorx/transports/clickhouse_arrow.rs

connectorx/transports/
clickhouse_arrow.rs
