duckdb Functions
This commit is contained in:
+135
-151
@@ -90,7 +90,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 55,
|
||||
"execution_count": 2,
|
||||
"id": "55c3cd57-0996-4723-beb5-8f3196c96009",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
@@ -104,7 +104,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 3,
|
||||
"id": "968403e3-2e5e-4834-b969-be4600e2963a",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
@@ -139,128 +139,10 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"id": "3634a4ec-04c2-4f1e-8659-5d22eb17a254",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>id</th>\n",
|
||||
" <th>from</th>\n",
|
||||
" <th>at</th>\n",
|
||||
" <th>to</th>\n",
|
||||
" <th>open</th>\n",
|
||||
" <th>close</th>\n",
|
||||
" <th>min</th>\n",
|
||||
" <th>max</th>\n",
|
||||
" <th>volume</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>999995</th>\n",
|
||||
" <td>7984748</td>\n",
|
||||
" <td>2023-03-03 18:13:30</td>\n",
|
||||
" <td>1677867225000000000</td>\n",
|
||||
" <td>2023-03-03 18:13:45</td>\n",
|
||||
" <td>1.062695</td>\n",
|
||||
" <td>1.062635</td>\n",
|
||||
" <td>1.062630</td>\n",
|
||||
" <td>1.062700</td>\n",
|
||||
" <td>64</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999996</th>\n",
|
||||
" <td>7984749</td>\n",
|
||||
" <td>2023-03-03 18:13:45</td>\n",
|
||||
" <td>1677867240000000000</td>\n",
|
||||
" <td>2023-03-03 18:14:00</td>\n",
|
||||
" <td>1.062645</td>\n",
|
||||
" <td>1.062650</td>\n",
|
||||
" <td>1.062625</td>\n",
|
||||
" <td>1.062650</td>\n",
|
||||
" <td>43</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999997</th>\n",
|
||||
" <td>7984750</td>\n",
|
||||
" <td>2023-03-03 18:14:00</td>\n",
|
||||
" <td>1677867255000000000</td>\n",
|
||||
" <td>2023-03-03 18:14:15</td>\n",
|
||||
" <td>1.062640</td>\n",
|
||||
" <td>1.062625</td>\n",
|
||||
" <td>1.062620</td>\n",
|
||||
" <td>1.062665</td>\n",
|
||||
" <td>47</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999998</th>\n",
|
||||
" <td>7984751</td>\n",
|
||||
" <td>2023-03-03 18:14:15</td>\n",
|
||||
" <td>1677867270000000000</td>\n",
|
||||
" <td>2023-03-03 18:14:30</td>\n",
|
||||
" <td>1.062625</td>\n",
|
||||
" <td>1.062535</td>\n",
|
||||
" <td>1.062535</td>\n",
|
||||
" <td>1.062645</td>\n",
|
||||
" <td>43</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>999999</th>\n",
|
||||
" <td>7984752</td>\n",
|
||||
" <td>2023-03-03 18:14:30</td>\n",
|
||||
" <td>1677867285000000000</td>\n",
|
||||
" <td>2023-03-03 18:14:45</td>\n",
|
||||
" <td>1.062535</td>\n",
|
||||
" <td>1.062520</td>\n",
|
||||
" <td>1.062520</td>\n",
|
||||
" <td>1.062580</td>\n",
|
||||
" <td>59</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" id from at \\\n",
|
||||
"999995 7984748 2023-03-03 18:13:30 1677867225000000000 \n",
|
||||
"999996 7984749 2023-03-03 18:13:45 1677867240000000000 \n",
|
||||
"999997 7984750 2023-03-03 18:14:00 1677867255000000000 \n",
|
||||
"999998 7984751 2023-03-03 18:14:15 1677867270000000000 \n",
|
||||
"999999 7984752 2023-03-03 18:14:30 1677867285000000000 \n",
|
||||
"\n",
|
||||
" to open close min max volume \n",
|
||||
"999995 2023-03-03 18:13:45 1.062695 1.062635 1.062630 1.062700 64 \n",
|
||||
"999996 2023-03-03 18:14:00 1.062645 1.062650 1.062625 1.062650 43 \n",
|
||||
"999997 2023-03-03 18:14:15 1.062640 1.062625 1.062620 1.062665 47 \n",
|
||||
"999998 2023-03-03 18:14:30 1.062625 1.062535 1.062535 1.062645 43 \n",
|
||||
"999999 2023-03-03 18:14:45 1.062535 1.062520 1.062520 1.062580 59 "
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %%time\n",
|
||||
"# Load Dataset\n",
|
||||
@@ -1486,6 +1368,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "50d1fc58-89a7-4507-aff0-6e943656cfe0",
|
||||
"metadata": {
|
||||
"jp-MarkdownHeadingCollapsed": true,
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
@@ -1632,49 +1515,82 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 80,
|
||||
"execution_count": 61,
|
||||
"id": "bbcdb883-d6dc-46db-88db-4c90b84522ba",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def duckdbConnect():\n",
|
||||
" cursor = duckdb.connect()\n",
|
||||
" print(cursor.execute(\"SELECT 42\").fetchall())\n",
|
||||
" return cursor"
|
||||
" return cursor\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def duckdbLoadCsv(csvFile=\"out.csv\"):\n",
|
||||
" data = pd.read_csv(\"out.csv\")\n",
|
||||
" return data\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# write\n",
|
||||
"def duckdbWrite(file, dataframe, tableName):\n",
|
||||
" conn = duckdbConnect()\n",
|
||||
" conn = duckdb.connect(file)\n",
|
||||
" # conn.execute(\"DROP TABLE EURUSDtest\")\n",
|
||||
" # conn.sql('CREATE TABLE EURUSDtest(i INTEGER)')\n",
|
||||
" conn.register(\"tempTable\", dataframe)\n",
|
||||
" conn.execute(\"CREATE TABLE {} AS SELECT * FROM tempTable\".format(tableName))\n",
|
||||
" conn.close()\n",
|
||||
" return 0\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def duckdbRead():\n",
|
||||
" conn = duckdbConnect()\n",
|
||||
" conn = duckdb.connect(file)\n",
|
||||
" conn.execute(\"SELECT * FROM EURUSD\").fetchdf()\n",
|
||||
" conn.close()\n",
|
||||
" return 0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 81,
|
||||
"id": "35025a6e-9dc7-46cf-a792-76b3d84f1ac0",
|
||||
"execution_count": 60,
|
||||
"id": "1c787f48-5640-4eb5-9456-be8f0a8211eb",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data = duckdbLoadCsv()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 68,
|
||||
"id": "f07f03d0-021e-4dc3-bfa8-efc029a9797a",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"CPU times: user 13.6 ms, sys: 151 µs, total: 13.7 ms\n",
|
||||
"Wall time: 12.7 ms\n"
|
||||
"ename": "CatalogException",
|
||||
"evalue": "Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mCatalogException\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[68], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m conn \u001b[38;5;241m=\u001b[39m duckdbConnect()\n\u001b[0;32m----> 2\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mDROP TABLE EURUSDtest\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[0;31mCatalogException\u001b[0m: Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"conn = duckdb.connect()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# in memory???\n",
|
||||
"def duckdbWrite():\n",
|
||||
" data = pd.read_csv(\"out.csv\")\n",
|
||||
" conn.register(\"EURUSDtest\", data)"
|
||||
"conn = duckdbConnect()\n",
|
||||
"conn.execute(\"DROP TABLE EURUSDtest\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 82,
|
||||
"id": "c6abdaaa-3ac2-425b-9208-d6cb79afe966",
|
||||
"execution_count": 71,
|
||||
"id": "9eb19431-fbad-43b4-84f7-0043e65de162",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -1704,14 +1620,27 @@
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>EURUSD</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>EURUSDtest</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>test</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"Empty DataFrame\n",
|
||||
"Columns: [name]\n",
|
||||
"Index: []"
|
||||
" name\n",
|
||||
"0 EURUSD\n",
|
||||
"1 EURUSDtest\n",
|
||||
"2 test"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -1719,13 +1648,15 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"conn = duckdb.connect(\"file.db\")\n",
|
||||
"# conn.sql(\"CREATE TABLE EURUSDtest(i INTEGER)\")\n",
|
||||
"display(conn.execute(\"SHOW TABLES\").df())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 76,
|
||||
"id": "2acce0f3-f0b2-47d0-8e0d-f9e9687efc18",
|
||||
"execution_count": 63,
|
||||
"id": "32037939-def0-48e7-9a47-46bcf1a5883c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
@@ -1737,17 +1668,70 @@
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mCatalogException\u001b[0m Traceback (most recent call last)",
|
||||
"File \u001b[0;32m<timed exec>:1\u001b[0m\n",
|
||||
"Cell \u001b[0;32mIn[63], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m conn \u001b[38;5;241m=\u001b[39m duckdb\u001b[38;5;241m.\u001b[39mconnect(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile.db\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 3\u001b[0m conn \u001b[38;5;241m=\u001b[39m duckdbConnect()\n\u001b[0;32m----> 4\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSELECT * FROM EURUSDtest\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 5\u001b[0m conn\u001b[38;5;241m.\u001b[39mfetchall()\n",
|
||||
"\u001b[0;31mCatalogException\u001b[0m: Catalog Error: Table with name EURUSDtest does not exist!\nDid you mean \"temp.information_schema.tables\"?\nLINE 1: SELECT * FROM EURUSDtest\n ^"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"df = conn.execute(\"SELECT * FROM EURUSDtest\").df()\n",
|
||||
"df"
|
||||
"# conn = duckdbConnect()\n",
|
||||
"conn.execute(\"SELECT * FROM EURUSDtest\")\n",
|
||||
"conn.fetchall()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 62,
|
||||
"id": "c6f53d67-684b-4b34-a573-472986ee3e47",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "CatalogException",
|
||||
"evalue": "Catalog Error: Table with name \"EURUSDtest\" already exists!",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mCatalogException\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[62], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m start \u001b[38;5;241m=\u001b[39m timeit\u001b[38;5;241m.\u001b[39mdefault_timer()\n\u001b[0;32m----> 2\u001b[0m dfDuckdb \u001b[38;5;241m=\u001b[39m \u001b[43mduckdbWrite\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfile.db\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdbname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m stop \u001b[38;5;241m=\u001b[39m timeit\u001b[38;5;241m.\u001b[39mdefault_timer()\n\u001b[1;32m 4\u001b[0m duckdb_write_execution_time \u001b[38;5;241m=\u001b[39m stop \u001b[38;5;241m-\u001b[39m start\n",
|
||||
"Cell \u001b[0;32mIn[61], line 18\u001b[0m, in \u001b[0;36mduckdbWrite\u001b[0;34m(file, dataframe, tableName)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;66;03m# conn.execute(\"DROP TABLE EURUSDtest\")\u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# conn.sql('CREATE TABLE EURUSDtest(i INTEGER)')\u001b[39;00m\n\u001b[1;32m 17\u001b[0m conn\u001b[38;5;241m.\u001b[39mregister(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtempTable\u001b[39m\u001b[38;5;124m\"\u001b[39m, dataframe)\n\u001b[0;32m---> 18\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mCREATE TABLE \u001b[39;49m\u001b[38;5;132;43;01m{}\u001b[39;49;00m\u001b[38;5;124;43m AS SELECT * FROM tempTable\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtableName\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 19\u001b[0m conn\u001b[38;5;241m.\u001b[39mclose()\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;241m0\u001b[39m\n",
|
||||
"\u001b[0;31mCatalogException\u001b[0m: Catalog Error: Table with name \"EURUSDtest\" already exists!"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"start = timeit.default_timer()\n",
|
||||
"dfDuckdb = duckdbWrite(\"file.db\", data, dbname)\n",
|
||||
"stop = timeit.default_timer()\n",
|
||||
"duckdb_write_execution_time = stop - start"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f630fc1a-0d52-4e3a-9dfe-1ec60d188033",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"start = timeit.default_timer()\n",
|
||||
"dfDuckdb = duckdbRead()\n",
|
||||
"stop = timeit.default_timer()\n",
|
||||
"duckdb_read_execution_time = stop - start"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c6abdaaa-3ac2-425b-9208-d6cb79afe966",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4409cc89-ed14-4313-ac89-65b826038533",
|
||||
|
||||
Reference in New Issue
Block a user