1243 lines
54 KiB
Plaintext
1243 lines
54 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "7f1f5a4a-ae34-4a27-9efa-399edc0e384a",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"## Benchmark: ClickHouse Vs. InfluxDB Vs. Postgresql Vs. Parquet \n",
|
||
"\n",
|
||
"-----\n",
|
||
"\n",
|
||
"#### How to use:\n",
|
||
"* Rename the file \"properties-model.ini\" to \"properties.ini\"\n",
|
||
"* Fill with your own credentials\n",
|
||
"----\n",
|
||
"\n",
|
||
"The proposal of this work is to compare the speed in read/writing a midle level of data ( a dataset with 9 columns and 50.000 lines) to four diferent databases:\n",
|
||
"* ClickHouse\n",
|
||
"* InfluxDB\n",
|
||
"* Postgresql\n",
|
||
"* Parquet (in a S3 Minio Storage) <br>\n",
|
||
"ToDo: <br>\n",
|
||
"* DuckDB with Polars\n",
|
||
"* MongoDB\n",
|
||
"* Kdb+\n",
|
||
"\n",
|
||
" \n",
|
||
"Deve-se relevar:\n",
|
||
"é uma \"cold-storage\" ou \"frezze-storage\"? <br>\n",
|
||
"influxdb: alta leitura e possui a vantagem da indexaçõa para vizualização de dados em gráficos.\n",
|
||
"\n",
|
||
"notas: \n",
|
||
"* comparar tamanho do csv com parquet"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "6bb26ce7-1e84-4665-accd-916bb977f95d",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### Imports "
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"id": "ab6c6c81-6ac1-4668-a79b-a9a0341fb35a",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"False"
|
||
]
|
||
},
|
||
"execution_count": 18,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"import configparser\n",
|
||
"from datetime import datetime\n",
|
||
"\n",
|
||
"import duckdb\n",
|
||
"import influxdb_client\n",
|
||
"import pandas as pd\n",
|
||
"\n",
|
||
"# import pymongo\n",
|
||
"from clickhouse_driver import Client\n",
|
||
"from dotenv import load_dotenv\n",
|
||
"from minio import Minio\n",
|
||
"from pymongo import MongoClient\n",
|
||
"from pytz import timezone\n",
|
||
"from sqlalchemy import create_engine\n",
|
||
"\n",
|
||
"load_dotenv()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "55c3cd57-0996-4723-beb5-8f3196c96009",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Variables\n",
|
||
"dbname = \"EURUSDtest\""
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "968403e3-2e5e-4834-b969-be4600e2963a",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"arq = configparser.RawConfigParser()\n",
|
||
"arq.read(\"properties.ini\")\n",
|
||
"ClickHouseUser = arq.get(\"CLICKHOUSE\", \"user\")\n",
|
||
"ClickHouseKey = arq.get(\"CLICKHOUSE\", \"key\")\n",
|
||
"ClickHouseUrl = arq.get(\"CLICKHOUSE\", \"url\")\n",
|
||
"\n",
|
||
"InfluxDBUser = arq.get(\"INFLUXDB\", \"user\")\n",
|
||
"InfluxDBKey = arq.get(\"INFLUXDB\", \"key\")\n",
|
||
"InfluxDBUrl = arq.get(\"INFLUXDB\", \"url\")\n",
|
||
"InfluxDBBucket = arq.get(\"INFLUXDB\", \"bucket\")\n",
|
||
"\n",
|
||
"PostgresqlUser = arq.get(\"POSTGRESQL\", \"user\")\n",
|
||
"PostgresqlKey = arq.get(\"POSTGRESQL\", \"key\")\n",
|
||
"PostgresqlUrl = arq.get(\"POSTGRESQL\", \"url\")\n",
|
||
"PostgresqlDB = arq.get(\"POSTGRESQL\", \"database\")\n",
|
||
"\n",
|
||
"S3MinioUser = arq.get(\"S3MINIO\", \"user\")\n",
|
||
"S3MinioKey = arq.get(\"S3MINIO\", \"key\")\n",
|
||
"S3MinioUrl = arq.get(\"S3MINIO\", \"url\")\n",
|
||
"S3MinioRegion = arq.get(\"S3MINIO\", \"region\")\n",
|
||
"\n",
|
||
"MongoUser = arq.get(\"MONGODB\", \"user\")\n",
|
||
"MongoKey = arq.get(\"MONGODB\", \"key\")\n",
|
||
"MongoUrl = arq.get(\"MONGODB\", \"url\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "3634a4ec-04c2-4f1e-8659-5d22eb17a254",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"# Load Dataset\n",
|
||
"df = pd.read_csv(\"out.csv\", index_col=0)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7e7c46b6-90ee-4ca3-8b5a-553b09ece913",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# df.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "76199f91-31d6-416b-9f15-5d435b3792c9",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"df[\"from\"] = pd.to_datetime(df[\"from\"], unit=\"s\")\n",
|
||
"df[\"to\"] = pd.to_datetime(df[\"to\"], unit=\"s\")\n",
|
||
"# Optional use when not transoformed yet\n",
|
||
"# Transform Datetime"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "274cc026-2f48-4e38-b80f-b1a9ff982060",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"#### Funçoes\n",
|
||
"\n",
|
||
"-> Class"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "27de1ec8-4de1-440a-b555-b4a46c5ef7ce",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def timestamp2dataHora(x, timezone_=\"America/Sao_Paulo\"):\n",
|
||
" d = datetime.fromtimestamp(x, tz=timezone(timezone_))\n",
|
||
" return d"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "4a8d5703-9bc9-4d38-83ff-457159304d58",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### ClickHouse"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9cf86669-7722-4a2c-895c-51f0a5eebefc",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# !! O client oficial usa um driver http, nesse exemplo vamos usar a biblioteca\n",
|
||
"# de terceirtos clickhouse_driver recomendada, por sua vez que usa tcp.\n",
|
||
"client = Client(\n",
|
||
" host=ClickHouseUrl,\n",
|
||
" user=ClickHouseUser,\n",
|
||
" password=ClickHouseKey,\n",
|
||
" settings={\"use_numpy\": True},\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a0a1f67b-2e63-462e-be66-d322d99837ea",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Create Tables in ClickHouse\n",
|
||
"# !! ALTERAR TIPOS !!\n",
|
||
"# ENGINE: 'Memory' desaparece quando server é reiniciado\n",
|
||
"client.execute(\n",
|
||
" \"CREATE TABLE IF NOT EXISTS {} (id UInt32,\"\n",
|
||
" \"from DateTime, at UInt64, to DateTime, open Float64,\"\n",
|
||
" \"close Float64, min Float64, max Float64, volume UInt32)\"\n",
|
||
" \"ENGINE MergeTree ORDER BY to\".format(dbname)\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "3a029a09-46f4-43c3-b3df-cfbed33fb0dc",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"# Write dataframe to db\n",
|
||
"client.insert_dataframe(\"INSERT INTO {} VALUES\".format(dbname), df)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "17251288-2442-43ee-98f2-ca680c3c4f13",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"client.query_dataframe(\"SELECT * FROM default.{}\".format(dbname)) # LIMIT 10000"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "51497522-bd6c-44a8-aaea-ec5dda30b95b",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# %%time\n",
|
||
"# df = pd.DataFrame(client.query_dataframe(\"SELECT * FROM default.{}\".format(dbname)))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "1d389546-911f-43f7-aad1-49f7bcc83503",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### InfluxDB\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c3e7ebfd-76f1-4ac4-9833-312eb1a531af",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"client = influxdb_client.InfluxDBClient(\n",
|
||
" url=InfluxDBUrl, token=InfluxDBKey, org=InfluxDBUser\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "cbf61f12-830b-4c57-804a-2257d8b3599a",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Read data from CSV without index and parse 'TimeStamp' as date.\n",
|
||
"df = pd.read_csv(\"out.csv\", sep=\",\", index_col=False, parse_dates=[\"from\"])\n",
|
||
"# Set 'TimeStamp' field as index of dataframe # test another indexs\n",
|
||
"df.set_index(\"from\", inplace=True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "54342a28-ba2b-4ade-a692-00566b53a639",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"df.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "f861fab2-f1b1-49dd-b758-12d10aef3462",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"# gravando... demorou... mas deu certo\n",
|
||
"with client.write_api() as writer:\n",
|
||
" writer.write(\n",
|
||
" bucket=InfluxDBBucket,\n",
|
||
" record=df,\n",
|
||
" data_frame_measurement_name=\"id\",\n",
|
||
" data_frame_tag_columns=[\"volume\"],\n",
|
||
" )"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "0bb2563d-68e2-4ff4-8842-70ac730dc6b1",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# data\n",
|
||
"# |> pivot(\n",
|
||
"# rowKey:[\"_time\"],\n",
|
||
"# columnKey: [\"_field\"],\n",
|
||
"# valueColumn: \"_value\"\n",
|
||
"# )"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "bb1596f9-4cee-4642-803a-ee61c9dddf64",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Read"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "b9ddfdc6-c899-4f6c-9b4e-8ec6ab6d7e05",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### Postgresql"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "16cd8eb7-333d-43fd-88e0-ee983645d3fd",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Connect / Create Tables\n",
|
||
"engine = create_engine(\n",
|
||
" \"postgresql+psycopg2://{}:{}@{}:5432/{}\".format(\n",
|
||
" PostgresqlUser, PostgresqlKey, PostgresqlUrl, PostgresqlDB\n",
|
||
" )\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "be31f3a0-b7ed-48e6-9b65-dc16319fb8d1",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Drop old table and create new empty table\n",
|
||
"df.head(0).to_sql(\"comparedbs\", engine, if_exists=\"replace\", index=False)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a7883c4d-4609-4380-8a45-246b7ca2f9c5",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"# Write\n",
|
||
"conn = engine.raw_connection()\n",
|
||
"cur = conn.cursor()\n",
|
||
"output = io.StringIO()\n",
|
||
"df.to_csv(output, sep=\"\\t\", header=False, index=False)\n",
|
||
"output.seek(0)\n",
|
||
"contents = output.getvalue()\n",
|
||
"\n",
|
||
"cur.copy_from(output, \"comparedbs\") # , null=\"\") # null values become ''\n",
|
||
"conn.commit()\n",
|
||
"cur.close()\n",
|
||
"conn.close()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "e37a93e1-fc0e-4d27-9e16-dca6c8aea324",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Read"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f9e0393d-7d1d-406a-a068-9dbf4968e977",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### S3 Parquet"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "60a990e2-4607-4654-84ec-17d4985adae2",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# fazer sem funçao para ver se melhora\n",
|
||
"# verifique se esta no ssd os arquivos da pasta git\n",
|
||
"def main():\n",
|
||
" client = Minio(\n",
|
||
" S3MinioUrl,\n",
|
||
" secure=False,\n",
|
||
" region=S3MinioRegion,\n",
|
||
" access_key=\"MatMPA7NyHltz7DQ\",\n",
|
||
" secret_key=\"SO1IG5iBPSjNPZanYUaHCLcoSbjphLCP\",\n",
|
||
" )\n",
|
||
"\n",
|
||
" # Make bucket if not exist.\n",
|
||
" found = client.bucket_exists(\"data\")\n",
|
||
" if not found:\n",
|
||
" client.make_bucket(\"data\")\n",
|
||
" else:\n",
|
||
" print(\"Bucket 'data' already exists\")\n",
|
||
"\n",
|
||
" # Upload\n",
|
||
" client.fput_object(\n",
|
||
" \"data\",\n",
|
||
" \"data.parquet\",\n",
|
||
" \"data/data.parquet\",\n",
|
||
" )\n",
|
||
" # print(\n",
|
||
" # \"'data/data.parquet' is successfully uploaded as \"\n",
|
||
" # \"object 'data.parquet' to bucket 'data'.\"\n",
|
||
" # )"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "390918c8-c88f-404a-96c4-685d578fdad0",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"df.to_parquet(\"data/data.parquet\")\n",
|
||
"if __name__ == \"__main__\":\n",
|
||
" try:\n",
|
||
" main()\n",
|
||
" except S3Error as exc:\n",
|
||
" print(\"error occurred.\", exc)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "a9e07143-8c11-4b68-a869-c3922cda9092",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"pq = pd.read_parquet(\"data/data.parquet\", engine=\"pyarrow\")\n",
|
||
"pq.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "50d1fc58-89a7-4507-aff0-6e943656cfe0",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### MongoDB"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "d104d9af-fa34-4261-8478-329a28ee4f2e",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Load csv dataset\n",
|
||
"data = pd.read_csv(\"out.csv\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 36,
|
||
"id": "0af8f72c-5b58-4dfc-af36-c5b4bc79f127",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# Connect to MongoDB\n",
|
||
"client = MongoClient(\n",
|
||
" # \"mongodb://192.168.1.133:27017\"\n",
|
||
" \"mongodb://{}:{}@{}/EURUSDtest?retryWrites=true&w=majority\".format(\n",
|
||
" MongoUser, MongoKey, MongoUrl\n",
|
||
" ),\n",
|
||
" authSource=\"admin\",\n",
|
||
")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 37,
|
||
"id": "f1b20d15-f5af-463c-813f-ffae61119de1",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"db = client[\"EUROUSDtest\"]\n",
|
||
"collection = db[\"finance\"]\n",
|
||
"# data.reset_index(inplace=True)\n",
|
||
"data_dict = data.to_dict(\"records\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 38,
|
||
"id": "70674d23-f375-4659-87ec-c745dec96d54",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"CPU times: user 19.2 s, sys: 269 ms, total: 19.5 s\n",
|
||
"Wall time: 50.1 s\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"<pymongo.results.InsertManyResult at 0x7f8e39c991e0>"
|
||
]
|
||
},
|
||
"execution_count": 38,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"%%time\n",
|
||
"# Insert collection\n",
|
||
"collection.insert_many(data_dict)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "81a4a33d-5914-45d8-af4e-2b0aabd2ac38",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# read"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "97405e42-61dc-42c7-8220-237a312c0ec7",
|
||
"metadata": {
|
||
"jp-MarkdownHeadingCollapsed": true,
|
||
"tags": []
|
||
},
|
||
"source": [
|
||
"### DuckDB"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 39,
|
||
"id": "bbcdb883-d6dc-46db-88db-4c90b84522ba",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[(42,)]\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"cursor = duckdb.connect()\n",
|
||
"print(cursor.execute(\"SELECT 42\").fetchall())"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 45,
|
||
"id": "35025a6e-9dc7-46cf-a792-76b3d84f1ac0",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"CPU times: user 1.53 s, sys: 63.6 ms, total: 1.59 s\n",
|
||
"Wall time: 1.59 s\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"<duckdb.DuckDBPyConnection at 0x7f8e3b2261b0>"
|
||
]
|
||
},
|
||
"execution_count": 45,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"%%time\n",
|
||
"conn = duckdb.connect()\n",
|
||
"data = pd.read_csv(\"out.csv\")\n",
|
||
"conn.register(\"EURUSDtest\", data)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 47,
|
||
"id": "c6abdaaa-3ac2-425b-9208-d6cb79afe966",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>name</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
"Empty DataFrame\n",
|
||
"Columns: [name]\n",
|
||
"Index: []"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"display(conn.execute(\"SHOW TABLES\").df())"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 46,
|
||
"id": "2acce0f3-f0b2-47d0-8e0d-f9e9687efc18",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>Unnamed: 0</th>\n",
|
||
" <th>id</th>\n",
|
||
" <th>from</th>\n",
|
||
" <th>at</th>\n",
|
||
" <th>to</th>\n",
|
||
" <th>open</th>\n",
|
||
" <th>close</th>\n",
|
||
" <th>min</th>\n",
|
||
" <th>max</th>\n",
|
||
" <th>volume</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>0</td>\n",
|
||
" <td>7730801</td>\n",
|
||
" <td>2023-01-02 15:58:45</td>\n",
|
||
" <td>1672675140000000000</td>\n",
|
||
" <td>2023-01-02 15:59:00</td>\n",
|
||
" <td>1.065995</td>\n",
|
||
" <td>1.066035</td>\n",
|
||
" <td>1.065930</td>\n",
|
||
" <td>1.066070</td>\n",
|
||
" <td>57</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>1</td>\n",
|
||
" <td>7730802</td>\n",
|
||
" <td>2023-01-02 15:59:00</td>\n",
|
||
" <td>1672675155000000000</td>\n",
|
||
" <td>2023-01-02 15:59:15</td>\n",
|
||
" <td>1.066055</td>\n",
|
||
" <td>1.066085</td>\n",
|
||
" <td>1.066005</td>\n",
|
||
" <td>1.066115</td>\n",
|
||
" <td>52</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>2</td>\n",
|
||
" <td>7730803</td>\n",
|
||
" <td>2023-01-02 15:59:15</td>\n",
|
||
" <td>1672675170000000000</td>\n",
|
||
" <td>2023-01-02 15:59:30</td>\n",
|
||
" <td>1.066080</td>\n",
|
||
" <td>1.066025</td>\n",
|
||
" <td>1.066025</td>\n",
|
||
" <td>1.066110</td>\n",
|
||
" <td>57</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>3</td>\n",
|
||
" <td>7730804</td>\n",
|
||
" <td>2023-01-02 15:59:30</td>\n",
|
||
" <td>1672675185000000000</td>\n",
|
||
" <td>2023-01-02 15:59:45</td>\n",
|
||
" <td>1.065980</td>\n",
|
||
" <td>1.065985</td>\n",
|
||
" <td>1.065885</td>\n",
|
||
" <td>1.066045</td>\n",
|
||
" <td>64</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>4</td>\n",
|
||
" <td>7730805</td>\n",
|
||
" <td>2023-01-02 15:59:45</td>\n",
|
||
" <td>1672675200000000000</td>\n",
|
||
" <td>2023-01-02 16:00:00</td>\n",
|
||
" <td>1.065975</td>\n",
|
||
" <td>1.066055</td>\n",
|
||
" <td>1.065830</td>\n",
|
||
" <td>1.066055</td>\n",
|
||
" <td>50</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>999995</th>\n",
|
||
" <td>999995</td>\n",
|
||
" <td>7984748</td>\n",
|
||
" <td>2023-03-03 18:13:30</td>\n",
|
||
" <td>1677867225000000000</td>\n",
|
||
" <td>2023-03-03 18:13:45</td>\n",
|
||
" <td>1.062695</td>\n",
|
||
" <td>1.062635</td>\n",
|
||
" <td>1.062630</td>\n",
|
||
" <td>1.062700</td>\n",
|
||
" <td>64</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>999996</th>\n",
|
||
" <td>999996</td>\n",
|
||
" <td>7984749</td>\n",
|
||
" <td>2023-03-03 18:13:45</td>\n",
|
||
" <td>1677867240000000000</td>\n",
|
||
" <td>2023-03-03 18:14:00</td>\n",
|
||
" <td>1.062645</td>\n",
|
||
" <td>1.062650</td>\n",
|
||
" <td>1.062625</td>\n",
|
||
" <td>1.062650</td>\n",
|
||
" <td>43</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>999997</th>\n",
|
||
" <td>999997</td>\n",
|
||
" <td>7984750</td>\n",
|
||
" <td>2023-03-03 18:14:00</td>\n",
|
||
" <td>1677867255000000000</td>\n",
|
||
" <td>2023-03-03 18:14:15</td>\n",
|
||
" <td>1.062640</td>\n",
|
||
" <td>1.062625</td>\n",
|
||
" <td>1.062620</td>\n",
|
||
" <td>1.062665</td>\n",
|
||
" <td>47</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>999998</th>\n",
|
||
" <td>999998</td>\n",
|
||
" <td>7984751</td>\n",
|
||
" <td>2023-03-03 18:14:15</td>\n",
|
||
" <td>1677867270000000000</td>\n",
|
||
" <td>2023-03-03 18:14:30</td>\n",
|
||
" <td>1.062625</td>\n",
|
||
" <td>1.062535</td>\n",
|
||
" <td>1.062535</td>\n",
|
||
" <td>1.062645</td>\n",
|
||
" <td>43</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>999999</th>\n",
|
||
" <td>999999</td>\n",
|
||
" <td>7984752</td>\n",
|
||
" <td>2023-03-03 18:14:30</td>\n",
|
||
" <td>1677867285000000000</td>\n",
|
||
" <td>2023-03-03 18:14:45</td>\n",
|
||
" <td>1.062535</td>\n",
|
||
" <td>1.062520</td>\n",
|
||
" <td>1.062520</td>\n",
|
||
" <td>1.062580</td>\n",
|
||
" <td>59</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>1000000 rows × 10 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" Unnamed: 0 id from at \n",
|
||
"0 0 7730801 2023-01-02 15:58:45 1672675140000000000 \\\n",
|
||
"1 1 7730802 2023-01-02 15:59:00 1672675155000000000 \n",
|
||
"2 2 7730803 2023-01-02 15:59:15 1672675170000000000 \n",
|
||
"3 3 7730804 2023-01-02 15:59:30 1672675185000000000 \n",
|
||
"4 4 7730805 2023-01-02 15:59:45 1672675200000000000 \n",
|
||
"... ... ... ... ... \n",
|
||
"999995 999995 7984748 2023-03-03 18:13:30 1677867225000000000 \n",
|
||
"999996 999996 7984749 2023-03-03 18:13:45 1677867240000000000 \n",
|
||
"999997 999997 7984750 2023-03-03 18:14:00 1677867255000000000 \n",
|
||
"999998 999998 7984751 2023-03-03 18:14:15 1677867270000000000 \n",
|
||
"999999 999999 7984752 2023-03-03 18:14:30 1677867285000000000 \n",
|
||
"\n",
|
||
" to open close min max volume \n",
|
||
"0 2023-01-02 15:59:00 1.065995 1.066035 1.065930 1.066070 57 \n",
|
||
"1 2023-01-02 15:59:15 1.066055 1.066085 1.066005 1.066115 52 \n",
|
||
"2 2023-01-02 15:59:30 1.066080 1.066025 1.066025 1.066110 57 \n",
|
||
"3 2023-01-02 15:59:45 1.065980 1.065985 1.065885 1.066045 64 \n",
|
||
"4 2023-01-02 16:00:00 1.065975 1.066055 1.065830 1.066055 50 \n",
|
||
"... ... ... ... ... ... ... \n",
|
||
"999995 2023-03-03 18:13:45 1.062695 1.062635 1.062630 1.062700 64 \n",
|
||
"999996 2023-03-03 18:14:00 1.062645 1.062650 1.062625 1.062650 43 \n",
|
||
"999997 2023-03-03 18:14:15 1.062640 1.062625 1.062620 1.062665 47 \n",
|
||
"999998 2023-03-03 18:14:30 1.062625 1.062535 1.062535 1.062645 43 \n",
|
||
"999999 2023-03-03 18:14:45 1.062535 1.062520 1.062520 1.062580 59 \n",
|
||
"\n",
|
||
"[1000000 rows x 10 columns]"
|
||
]
|
||
},
|
||
"execution_count": 46,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"%%time\n",
|
||
"df = conn.execute(\"SELECT * FROM EURUSDtest\").df()\n",
|
||
"df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "4409cc89-ed14-4313-ac89-65b826038533",
|
||
"metadata": {},
|
||
"source": [
|
||
"### Kdb+"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 66,
|
||
"id": "14f63810-1943-4e28-9bce-2148be6be02d",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import numpy as np\n",
|
||
"\n",
|
||
"np.bool = np.bool_\n",
|
||
"from qpython import qconnection"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 49,
|
||
"id": "8ff6c090-7e02-435a-a179-f2aab81da972",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# read csv\n",
|
||
"data = pd.read_csv(\"out.csv\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 50,
|
||
"id": "b4eb8ab9-81e8-4732-8cf7-51f0981d3d57",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# open connection\n",
|
||
"q = qconnection.QConnection(host=\"localhost\", port=5001)\n",
|
||
"q.open()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 51,
|
||
"id": "97cb6b5b-65a5-46a0-a4ee-e5c535a716ab",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"CPU times: user 837 ms, sys: 40 ms, total: 877 ms\n",
|
||
"Wall time: 1.16 s\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# send df to kd+ in memory bank\n",
|
||
"%%time\n",
|
||
"q.sendSync(\"{t::x}\", data)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 52,
|
||
"id": "c2ed2d51-bc8e-4207-892a-35fc55d43570",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"b':/home/sandman/q/tab1'"
|
||
]
|
||
},
|
||
"execution_count": 52,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# write to on disk table\n",
|
||
"q.sendSync(\"`:/home/sandman/q/tab1 set t\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 53,
|
||
"id": "9c055a95-f73f-43a3-8fbd-61e42235117e",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"CPU times: user 1.78 ms, sys: 2 µs, total: 1.79 ms\n",
|
||
"Wall time: 329 ms\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"%%time\n",
|
||
"# read from on disk table\n",
|
||
"df2 = q.sendSync(\"tab2: get `:/home/sandman/q/tab1\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 54,
|
||
"id": "9760de38-9f04-4322-bfff-c7ee12d5dee5",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"# print(df2)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 55,
|
||
"id": "c06c9222-c69d-4872-9d21-052281a013e2",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"CPU times: user 957 ms, sys: 87.9 ms, total: 1.05 s\n",
|
||
"Wall time: 1.13 s\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"%%time\n",
|
||
"# load to variable df2\n",
|
||
"df2 = q.sendSync(\"tab2\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 58,
|
||
"id": "8815f01c-fd0a-4f94-ab7f-f8ede84ba4e7",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"ename": "TypeError",
|
||
"evalue": "'QTable' object is not callable",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||
"Cell \u001b[0;32mIn[58], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdf2\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m)\u001b[49m\n",
|
||
"\u001b[0;31mTypeError\u001b[0m: 'QTable' object is not callable"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# df2(type)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 67,
|
||
"id": "e6ed3927-4395-45cd-9a28-88c5db01f2e5",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [
|
||
{
|
||
"ename": "AttributeError",
|
||
"evalue": "'bool' object has no attribute 'to_numpy'",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||
"File \u001b[0;32m<timed exec>:2\u001b[0m\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qconnection.py:385\u001b[0m, in \u001b[0;36mQConnection.__call__\u001b[0;34m(self, *parameters, **options)\u001b[0m\n\u001b[1;32m 384\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39mparameters, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions):\n\u001b[0;32m--> 385\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msendSync\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparameters\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparameters\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qconnection.py:303\u001b[0m, in \u001b[0;36mQConnection.sendSync\u001b[0;34m(self, query, *parameters, **options)\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''Performs a synchronous query against a q service and returns parsed \u001b[39;00m\n\u001b[1;32m 250\u001b[0m \u001b[38;5;124;03mdata.\u001b[39;00m\n\u001b[1;32m 251\u001b[0m \u001b[38;5;124;03m\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[38;5;124;03m :class:`.QReaderException`\u001b[39;00m\n\u001b[1;32m 301\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 302\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquery(MessageType\u001b[38;5;241m.\u001b[39mSYNC, query, \u001b[38;5;241m*\u001b[39mparameters, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions)\n\u001b[0;32m--> 303\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreceive\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_only\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mtype \u001b[38;5;241m==\u001b[39m MessageType\u001b[38;5;241m.\u001b[39mRESPONSE:\n\u001b[1;32m 306\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\u001b[38;5;241m.\u001b[39mdata\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qconnection.py:380\u001b[0m, in \u001b[0;36mQConnection.receive\u001b[0;34m(self, data_only, **options)\u001b[0m\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mreceive\u001b[39m(\u001b[38;5;28mself\u001b[39m, data_only \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions):\n\u001b[1;32m 342\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m'''Reads and (optionally) parses the response from a q service.\u001b[39;00m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;124;03m \u001b[39;00m\n\u001b[1;32m 344\u001b[0m \u001b[38;5;124;03m Retrieves query result along with meta-information:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[38;5;124;03m :raises: :class:`.QReaderException`\u001b[39;00m\n\u001b[1;32m 379\u001b[0m \u001b[38;5;124;03m '''\u001b[39;00m\n\u001b[0;32m--> 380\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_options\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munion_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 381\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\u001b[38;5;241m.\u001b[39mdata \u001b[38;5;28;01mif\u001b[39;00m data_only \u001b[38;5;28;01melse\u001b[39;00m result\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qreader.py:139\u001b[0m, in \u001b[0;36mQReader.read\u001b[0;34m(self, source, **options)\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 121\u001b[0m \u001b[38;5;124;03mReads and optionally parses a single message.\u001b[39;00m\n\u001b[1;32m 122\u001b[0m \u001b[38;5;124;03m\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;124;03m with meta information\u001b[39;00m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 138\u001b[0m message \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_header(source)\n\u001b[0;32m--> 139\u001b[0m message\u001b[38;5;241m.\u001b[39mdata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mis_compressed\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m message\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qreader.py:216\u001b[0m, in \u001b[0;36mQReader.read_data\u001b[0;34m(self, message_size, is_compressed, **options)\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_stream \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_options\u001b[38;5;241m.\u001b[39mraw:\n\u001b[1;32m 214\u001b[0m raw_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer\u001b[38;5;241m.\u001b[39mraw(message_size \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m8\u001b[39m)\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m raw_data \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_options\u001b[38;5;241m.\u001b[39mraw \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_object\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qreader.py:225\u001b[0m, in \u001b[0;36mQReader._read_object\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 222\u001b[0m reader \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_reader(qtype)\n\u001b[1;32m 224\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m reader:\n\u001b[0;32m--> 225\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mreader\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 226\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m qtype \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m QBOOL_LIST \u001b[38;5;129;01mand\u001b[39;00m qtype \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m QTIME_LIST:\n\u001b[1;32m 227\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_list(qtype)\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/_pandas.py:76\u001b[0m, in \u001b[0;36mPandasQReader._read_table\u001b[0;34m(self, qtype)\u001b[0m\n\u001b[1;32m 74\u001b[0m columns \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_object()\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer\u001b[38;5;241m.\u001b[39mskip() \u001b[38;5;66;03m# ignore generic list type indicator\u001b[39;00m\n\u001b[0;32m---> 76\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[43mQReader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_general_list\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 78\u001b[0m odict \u001b[38;5;241m=\u001b[39m OrderedDict()\n\u001b[1;32m 79\u001b[0m meta \u001b[38;5;241m=\u001b[39m MetaData(qtype \u001b[38;5;241m=\u001b[39m QTABLE)\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qreader.py:338\u001b[0m, in \u001b[0;36mQReader._read_general_list\u001b[0;34m(self, qtype)\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer\u001b[38;5;241m.\u001b[39mskip() \u001b[38;5;66;03m# ignore attributes\u001b[39;00m\n\u001b[1;32m 336\u001b[0m length \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer\u001b[38;5;241m.\u001b[39mget_int()\n\u001b[0;32m--> 338\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_object() \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(length)]\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qreader.py:338\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 335\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer\u001b[38;5;241m.\u001b[39mskip() \u001b[38;5;66;03m# ignore attributes\u001b[39;00m\n\u001b[1;32m 336\u001b[0m length \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer\u001b[38;5;241m.\u001b[39mget_int()\n\u001b[0;32m--> 338\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_object\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(length)]\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/qreader.py:227\u001b[0m, in \u001b[0;36mQReader._read_object\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m reader(\u001b[38;5;28mself\u001b[39m, qtype)\n\u001b[1;32m 226\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m qtype \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m QBOOL_LIST \u001b[38;5;129;01mand\u001b[39;00m qtype \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m QTIME_LIST:\n\u001b[0;32m--> 227\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_list\u001b[49m\u001b[43m(\u001b[49m\u001b[43mqtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 228\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m qtype \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m QBOOL \u001b[38;5;129;01mand\u001b[39;00m qtype \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m QTIME:\n\u001b[1;32m 229\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_atom(qtype)\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/qpython/_pandas.py:116\u001b[0m, in \u001b[0;36mPandasQReader._read_list\u001b[0;34m(self, qtype)\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;241m-\u001b[39m\u001b[38;5;28mabs\u001b[39m(qtype) \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]:\n\u001b[1;32m 115\u001b[0m null \u001b[38;5;241m=\u001b[39m QNULLMAP[\u001b[38;5;241m-\u001b[39m\u001b[38;5;28mabs\u001b[39m(qtype)][\u001b[38;5;241m1\u001b[39m]\n\u001b[0;32m--> 116\u001b[0m ps \u001b[38;5;241m=\u001b[39m \u001b[43mpandas\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSeries\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mqlist\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreplace\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnull\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumpy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mNaN\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 118\u001b[0m ps \u001b[38;5;241m=\u001b[39m pandas\u001b[38;5;241m.\u001b[39mSeries(data \u001b[38;5;241m=\u001b[39m qlist)\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/pandas/core/series.py:5219\u001b[0m, in \u001b[0;36mSeries.replace\u001b[0;34m(self, to_replace, value, inplace, limit, regex, method)\u001b[0m\n\u001b[1;32m 5203\u001b[0m \u001b[38;5;129m@doc\u001b[39m(\n\u001b[1;32m 5204\u001b[0m NDFrame\u001b[38;5;241m.\u001b[39mreplace,\n\u001b[1;32m 5205\u001b[0m klass\u001b[38;5;241m=\u001b[39m_shared_doc_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mklass\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 5217\u001b[0m method: Literal[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpad\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mffill\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbfill\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m|\u001b[39m lib\u001b[38;5;241m.\u001b[39mNoDefault \u001b[38;5;241m=\u001b[39m lib\u001b[38;5;241m.\u001b[39mno_default,\n\u001b[1;32m 5218\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Series \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 5219\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreplace\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 5220\u001b[0m \u001b[43m \u001b[49m\u001b[43mto_replace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mto_replace\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5221\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5222\u001b[0m \u001b[43m \u001b[49m\u001b[43minplace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minplace\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5223\u001b[0m \u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlimit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5224\u001b[0m \u001b[43m \u001b[49m\u001b[43mregex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mregex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5225\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5226\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/pandas/core/generic.py:7389\u001b[0m, in \u001b[0;36mNDFrame.replace\u001b[0;34m(self, to_replace, value, inplace, limit, regex, method)\u001b[0m\n\u001b[1;32m 7383\u001b[0m new_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mgr\u001b[38;5;241m.\u001b[39mreplace_regex(\n\u001b[1;32m 7384\u001b[0m to_replace\u001b[38;5;241m=\u001b[39mto_replace,\n\u001b[1;32m 7385\u001b[0m value\u001b[38;5;241m=\u001b[39mvalue,\n\u001b[1;32m 7386\u001b[0m inplace\u001b[38;5;241m=\u001b[39minplace,\n\u001b[1;32m 7387\u001b[0m )\n\u001b[1;32m 7388\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 7389\u001b[0m new_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mgr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreplace\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7390\u001b[0m \u001b[43m \u001b[49m\u001b[43mto_replace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mto_replace\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minplace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minplace\u001b[49m\n\u001b[1;32m 7391\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7392\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 7393\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 7394\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mInvalid \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mto_replace\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m type: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mrepr\u001b[39m(\u001b[38;5;28mtype\u001b[39m(to_replace)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 7395\u001b[0m )\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/pandas/core/internals/managers.py:475\u001b[0m, in \u001b[0;36mBaseBlockManager.replace\u001b[0;34m(self, to_replace, value, inplace)\u001b[0m\n\u001b[1;32m 473\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_list_like(to_replace)\n\u001b[1;32m 474\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_list_like(value)\n\u001b[0;32m--> 475\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 476\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mreplace\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 477\u001b[0m \u001b[43m \u001b[49m\u001b[43mto_replace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mto_replace\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 478\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 479\u001b[0m \u001b[43m \u001b[49m\u001b[43minplace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minplace\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 480\u001b[0m \u001b[43m \u001b[49m\u001b[43musing_cow\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43musing_copy_on_write\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 481\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/pandas/core/internals/managers.py:352\u001b[0m, in \u001b[0;36mBaseBlockManager.apply\u001b[0;34m(self, f, align_keys, **kwargs)\u001b[0m\n\u001b[1;32m 350\u001b[0m applied \u001b[38;5;241m=\u001b[39m b\u001b[38;5;241m.\u001b[39mapply(f, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 351\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 352\u001b[0m applied \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 353\u001b[0m result_blocks \u001b[38;5;241m=\u001b[39m extend_blocks(applied, result_blocks)\n\u001b[1;32m 355\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39mfrom_blocks(result_blocks, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxes)\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/pandas/core/internals/blocks.py:593\u001b[0m, in \u001b[0;36mBlock.replace\u001b[0;34m(self, to_replace, value, inplace, mask, using_cow)\u001b[0m\n\u001b[1;32m 590\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[38;5;28mself\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m inplace \u001b[38;5;28;01melse\u001b[39;00m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcopy()]\n\u001b[1;32m 592\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mask \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 593\u001b[0m mask \u001b[38;5;241m=\u001b[39m \u001b[43mmissing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmask_missing\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mto_replace\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 594\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m mask\u001b[38;5;241m.\u001b[39many():\n\u001b[1;32m 595\u001b[0m \u001b[38;5;66;03m# Note: we get here with test_replace_extension_other incorrectly\u001b[39;00m\n\u001b[1;32m 596\u001b[0m \u001b[38;5;66;03m# bc _can_hold_element is incorrect.\u001b[39;00m\n\u001b[1;32m 597\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m using_cow:\n",
|
||
"File \u001b[0;32m~/dev/pipenv/lib/python3.10/site-packages/pandas/core/missing.py:112\u001b[0m, in \u001b[0;36mmask_missing\u001b[0;34m(arr, values_to_mask)\u001b[0m\n\u001b[1;32m 108\u001b[0m new_mask \u001b[38;5;241m=\u001b[39m arr \u001b[38;5;241m==\u001b[39m x\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_mask, np\u001b[38;5;241m.\u001b[39mndarray):\n\u001b[1;32m 111\u001b[0m \u001b[38;5;66;03m# usually BooleanArray\u001b[39;00m\n\u001b[0;32m--> 112\u001b[0m new_mask \u001b[38;5;241m=\u001b[39m \u001b[43mnew_mask\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_numpy\u001b[49m(dtype\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mbool\u001b[39m, na_value\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 113\u001b[0m mask \u001b[38;5;241m|\u001b[39m\u001b[38;5;241m=\u001b[39m new_mask\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m na_mask\u001b[38;5;241m.\u001b[39many():\n",
|
||
"\u001b[0;31mAttributeError\u001b[0m: 'bool' object has no attribute 'to_numpy'"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"%%time\n",
|
||
"# converto to dataframe\n",
|
||
"df = pd.DataFrame(q(\"t\", pandas=True))\n",
|
||
"df.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 19,
|
||
"id": "0fc7f16b-6c39-4ebe-88d2-ff857e30ab62",
|
||
"metadata": {
|
||
"tags": []
|
||
},
|
||
"outputs": [],
|
||
"source": [
|
||
"%%time\n",
|
||
"# select\n",
|
||
"df3 = q.sendSync(\"select from t\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c88646ca-3d25-4a85-80b5-f9e559f568dd",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"q.close()"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.10.11"
|
||
},
|
||
"widgets": {
|
||
"application/vnd.jupyter.widget-state+json": {
|
||
"state": {},
|
||
"version_major": 2,
|
||
"version_minor": 0
|
||
}
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|