rust 无法将json转换为dataframe，polars恐慌

oknrviil 于 2023-06-30 发布在其他

关注(0)|答案(1)|浏览(157)

use reqwest::blocking::get;
use polars::prelude::*;
use serde::{Deserialize, Serialize};
use std::io::Cursor;

#[derive(Serialize, Deserialize, Debug)]
struct StockZhAHist {
    date: String,
    open: f64,
    close: f64,
    high: f64,
    low: f64,
    volume: f64,
    turnover: f64,
    amplitude: f64,
    change_rate: f64,
    change_amount: f64,
    turnover_rate: f64,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let url = "http://127.0.0.1:8080/api/public/stock_zh_a_hist";
    let response = get(url)?.text()?;
    // println!("{}", response);
    // deserialize response with serde
    let stock_zh_a_hist: Vec<StockZhAHist> = serde_json::from_str(&response)?;
    // convert to polars dataframe
    println!("{:?}", stock_zh_a_hist);
    let df = JsonReader::new(Cursor::new(response))
        .infer_schema_len(Some(1000))
        .finish()?;
    println!("{:?}", df);
    Ok(())
}

我写了一个脚本，从我的服务器检索原始数据，我能够打印出JSON 'stock_zh_a_hist'。但不知何故，当我试图打印出从这个JSON转换的dataframe时，程序死机并返回：
线程“main”在“Arrow datatype Struct（[Field { name：“date”，data_type：LargeUtf8，is_nullable：true，元数据：{} }，字段{ name：“open”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“close”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“high”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“low”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“volume”，data_type：Int64，is_nullable：true，元数据：{} }，字段{ name：“turnover”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“amplitude”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“change_rate”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“change_amount”，data_type：Float64，is_nullable：true，元数据：{} }，字段{ name：“turnover_rate”，data_type：Float64，is_nullable：true，元数据：{} }]）不受Polars支持。您可能需要激活该数据类型功能。'，/home/亚瑟/.cargo/registry/src/index.crates.io-6 f17 d22 bba 15001 f/polars-core-0.30.0/src/datatypes/field.rs：158：19.
我还读了源代码，内容如下：

#[cfg(feature = "dtype-decimal")]
            ArrowDataType::Decimal(precision, scale) => DataType::Decimal(Some(*precision), Some(*scale)),
            dt => panic!("Arrow datatype {dt:?} not supported by Polars. You probably need to activate that data-type feature."),

我假设polars在阅读正确的数据类型的json时遇到了一些问题，但是我如何解决呢？

更新1：我已经在cargo.toml中添加了dtype-decimal特性，但是同样的错误仍然存在。
更新2：stock_zh_a_hist的内容如下：

[StockZhAHist { date: "1996-12-16T00:00:00.000", open: 16.86, close: 16.86, high: 16.86, low: 16.86, volume: 62442.0, turnover: 105277000.0, amplitude: 0.0, change_rate: -10.22, change_amount: -1.92, turnover_rate: 0.87 },
StockZhAHist { date: "1996-12-17T00:00:00.000", open: 15.17, close: 15.17, high: 16.79, low: 15.17, volume: 463675.0, turnover: 718902016.0, amplitude: 9.61, change_rate: -10.02, change_amount: -1.69, turnover_rate: 6.49 },
StockZhAHist { date: "1996-12-18T00:00:00.000", open: 15.28, close: 16.69, high: 16.69, low: 15.18, volume: 445380.0, turnover: 719400000.0, amplitude: 9.95, change_rate: 10.02, change_amount: 1.52, turnover_rate: 6.24 },
StockZhAHist { date: "1996-12-19T00:00:00.000", open: 17.01, close: 16.4, high: 17.9, low: 15.99, volume: 572946.0, turnover: 970124992.0, amplitude: 11.44, change_rate: -1.74, change_amount: -0.29, turnover_rate: 8.03 },
StockZhAHist { date: "1996-12-20T00:00:00.000", open: 16.19, close: 16.39, high: 16.68, low: 15.9, volume: 277758.0, turnover: 451471008.0, amplitude: 4.76, change_rate: -0.06, change_amount: -0.01, turnover_rate: 3.89 },
StockZhAHist { date: "1996-12-23T00:00:00.000", open: 16.58, close: 16.57, high: 16.89, low: 16.4, volume: 155369.0, turnover: 258260992.0, amplitude: 2.99, change_rate: 1.1, change_amount: 0.18, turnover_rate: 2.18 },
StockZhAHist { date: "1996-12-24T00:00:00.000", open: 16.58, close: 15.95, high: 16.73, low: 15.93, volume: 153168.0, turnover: 249324000.0, amplitude: 4.83, change_rate: -3.74, change_amount: -0.62, turnover_rate: 2.15 },
StockZhAHist { date: "1996-12-25T00:00:00.000", open: 16.1, close: 16.47, high: 16.7, low: 15.86, volume: 175391.0, turnover: 285567008.0, amplitude: 5.27, change_rate: 3.26, change_amount: 0.52, turnover_rate: 2.46 },
StockZhAHist { date: "1996-12-26T00:00:00.000", open: 16.68, close: 16.47, high: 16.78, low: 16.3, volume: 51516.0, turnover: 85214000.0, amplitude: 2.91, change_rate: 0.0, change_amount: 0.0, turnover_rate: 0.72 }...]

rust

来源：https://stackoverflow.com/questions/76563450/unable-to-convert-json-to-dataframe-polars-panicked

1条答案

按热度按时间

beq87vna1#

看起来像是一个bug --如果你启用了dtype-struct特性，这个问题就会消失，但是你绝对不应该为了从json中读取而必须这样做。
作为一种解决方法，即使没有dtype-struct，也可以通过将数据转换为JSON行来实现这一点。

use polars::prelude::{JsonReader, SerReader};
use serde::{Deserialize, Serialize};
use std::io::Cursor;

#[derive(Serialize, Deserialize, Debug)]
struct StockZhAHist {
    date: String,
    open: f64,
    close: f64,
    high: f64,
    low: f64,
    volume: f64,
    turnover: f64,
    amplitude: f64,
    change_rate: f64,
    change_amount: f64,
    turnover_rate: f64,
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let data = vec![
        StockZhAHist {
            date: "1996-12-16T00:00:00.000".into(),
            open: 16.86,
            close: 16.86,
            high: 16.86,
            low: 16.86,
            volume: 62442.0,
            turnover: 105277000.0,
            amplitude: 0.0,
            change_rate: -10.22,
            change_amount: -1.92,
            turnover_rate: 0.87,
        },
        StockZhAHist {
            date: "1996-12-17T00:00:00.000".into(),
            open: 15.17,
            close: 15.17,
            high: 16.79,
            low: 15.17,
            volume: 463675.0,
            turnover: 718902016.0,
            amplitude: 9.61,
            change_rate: -10.02,
            change_amount: -1.69,
            turnover_rate: 6.49,
        },
        StockZhAHist {
            date: "1996-12-18T00:00:00.000".into(),
            open: 15.28,
            close: 16.69,
            high: 16.69,
            low: 15.18,
            volume: 445380.0,
            turnover: 719400000.0,
            amplitude: 9.95,
            change_rate: 10.02,
            change_amount: 1.52,
            turnover_rate: 6.24,
        },
    ];
    // JSON Lines data
    let response = data
        .into_iter()
        .map(|d| serde_json::to_string(&d))
        .collect::<Result<Vec<_>, _>>()?
        .join("\n");
    println!("{}", response);
    // convert to polars dataframe
    let df = JsonReader::new(Cursor::new(response))
        .infer_schema_len(Some(1000))
        .with_json_format(polars::prelude::JsonFormat::JsonLines)
        .finish()?;
    println!("{:?}", df);
    Ok(())
}

{"date":"1996-12-16T00:00:00.000","open":16.86,"close":16.86,"high":16.86,"low":16.86,"volume":62442.0,"turnover":105277000.0,"amplitude":0.0,"change_rate":-10.22,"change_amount":-1.92,"turnover_rate":0.87}
{"date":"1996-12-17T00:00:00.000","open":15.17,"close":15.17,"high":16.79,"low":15.17,"volume":463675.0,"turnover":718902016.0,"amplitude":9.61,"change_rate":-10.02,"change_amount":-1.69,"turnover_rate":6.49}
{"date":"1996-12-18T00:00:00.000","open":15.28,"close":16.69,"high":16.69,"low":15.18,"volume":445380.0,"turnover":719400000.0,"amplitude":9.95,"change_rate":10.02,"change_amount":1.52,"turnover_rate":6.24}
[/Users/ben/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polars-io-0.30.0/src/ndjson/core.rs:162] &data_type = Struct(
    [
        Field {
            name: "date",
            data_type: LargeUtf8,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "open",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "close",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "high",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "low",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "volume",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "turnover",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "amplitude",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "change_rate",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "change_amount",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
        Field {
            name: "turnover_rate",
            data_type: Float64,
            is_nullable: true,
            metadata: {},
        },
    ],
)
shape: (3, 11)
┌─────────────────────────┬───────┬───────┬───────┬───┬───────────┬─────────────┬───────────────┬───────────────┐
│ date                    ┆ open  ┆ close ┆ high  ┆ … ┆ amplitude ┆ change_rate ┆ change_amount ┆ turnover_rate │
│ ---                     ┆ ---   ┆ ---   ┆ ---   ┆   ┆ ---       ┆ ---         ┆ ---           ┆ ---           │
│ str                     ┆ f64   ┆ f64   ┆ f64   ┆   ┆ f64       ┆ f64         ┆ f64           ┆ f64           │
╞═════════════════════════╪═══════╪═══════╪═══════╪═══╪═══════════╪═════════════╪═══════════════╪═══════════════╡
│ 1996-12-16T00:00:00.000 ┆ 16.86 ┆ 16.86 ┆ 16.86 ┆ … ┆ 0.0       ┆ -10.22      ┆ -1.92         ┆ 0.87          │
│ 1996-12-17T00:00:00.000 ┆ 15.17 ┆ 15.17 ┆ 16.79 ┆ … ┆ 9.61      ┆ -10.02      ┆ -1.69         ┆ 6.49          │
│ 1996-12-18T00:00:00.000 ┆ 15.28 ┆ 16.69 ┆ 16.69 ┆ … ┆ 9.95      ┆ 10.02       ┆ 1.52          ┆ 6.24          │
└─────────────────────────┴───────┴───────┴───────┴───┴───────────┴─────────────┴───────────────┴───────────────┘

赞(0）回复(0）举报 2023-06-30

我来回答

rust 无法将json转换为dataframe，polars恐慌

1条答案

相关问题

热门标签

最新问答