Arrow -> PostgreSQL encoder
Project description
pgpq
Convert PyArrow RecordBatches to Postgres' native binary format.
Usage
import psycopg
import pyarrow.dataset as ds
from pgpq import ArrowToPostgresBinaryEncoder
dataset = ds.dataset("tests/testdata/")
encoder = ArrowToPostgresBinaryEncoder(dataset.schema)
# get the expected postgres destination schema
# note that this is _not_ the same as the incoming arrow schema
# and not necessarily the schema of your permanent table
# instead it's the schema of the data that will be sent over the wire
# which for example does not have timezones on any timestamps
pg_schema = encoder.schema()
# assemble ddl for a temporary table
# it's often a good idea to bulk load into a temp table to:
# (1) Avoid indexes
# (2) Stay in-memory as long as possible
# (3) Be more flexible with types (you can't load a SMALLINT into a BIGINT column without casting)
cols = [f"\"{col['name']}\" {col['data_type']['ddl']}" for col in pg_schema["columns"]]
ddl = f"CREATE TEMP TABLE data ({','.join(cols)})"
with psycopg.connect("postgres://postgres:postgres@localhost:5432/postgres") as conn:
with conn.cursor() as cursor:
cursor.execute(ddl) # type: ignore
with cursor.copy("COPY data FROM STDIN WITH (FORMAT BINARY)") as copy:
copy.write(encoder.write_header())
for batch in dataset.to_batches():
copy.write(encoder.write_batch(batch))
copy.write(encoder.finish())
# load into your actual table, possibly doing type casts
# cursor.execute("INSERT INTO \"table\" SELECT * FROM data")
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
pgpq-0.6.1.tar.gz
(33.0 kB
view hashes)
Built Distributions
pgpq-0.6.1-cp37-abi3-win_amd64.whl
(502.5 kB
view hashes)
pgpq-0.6.1-cp37-abi3-win32.whl
(455.3 kB
view hashes)
Close
Hashes for pgpq-0.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 68b9333ca66fa4f88b2b2a04f248758dc60b5793ffead508a7345c38a27ad48a |
|
MD5 | 3d6f009fdd2b45f950901fdf13b7ead3 |
|
BLAKE2b-256 | 8349e0b80bd4285f5eb70cc1dea6cbf721535c2c1fa3b2166371d3a65ceb1e39 |
Close
Hashes for pgpq-0.6.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | b124885c6d5d8bc9c83a965c4cbf521fe813af98b5bbe9210b880f8fbe387d26 |
|
MD5 | e298fd7117b55e94260ea214a9400e4e |
|
BLAKE2b-256 | 7a3c4227dbc85999a616c7152a6090a39f9ce7cf68427f9896fa875fec7a90c1 |
Close
Hashes for pgpq-0.6.1-pp39-pypy39_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | eb4fca4e18cad4357fe95cc98816c893484262685753aa630b84df1e87c97430 |
|
MD5 | ee6ed78dde62b3f27cfc87fd19f65e8e |
|
BLAKE2b-256 | 7ab0ab8bbfd92b0649ad2516090b161282ec60ef3efbb62a5735e2efeae6c588 |
Close
Hashes for pgpq-0.6.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 374d7adec12d6e29236390e09d0fbc1fa9ec253fe903c6d4595b99c6464dc21f |
|
MD5 | e00d766d6278dd4b102486d0ec45be96 |
|
BLAKE2b-256 | 03cb1c31e8779cefcca9b385425f6935c045802e379588ac0d2f0af793a69642 |
Close
Hashes for pgpq-0.6.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 917d85a4e546555e9d42070e4addd3c8d2a8b4eb0c6e6f0f91237045b134115c |
|
MD5 | ea32e92eb95defc97512034ccb8d5d30 |
|
BLAKE2b-256 | cd34bf52dbf3a2e2ad17928f31572cbff14dd3fdc53cc5b7b9cf0875cdead2e3 |
Close
Hashes for pgpq-0.6.1-pp38-pypy38_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | bc34761e963d2a8a81c90a21adfd5a6172fcf12b89b4e4b93a4a30a66875df84 |
|
MD5 | de5d9babdf21e8ceb30e181f26262c78 |
|
BLAKE2b-256 | 447b64dd635bb09c109dcd8433908ccb347f857b4c6d68323848073e3903bd76 |
Close
Hashes for pgpq-0.6.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | dd1563b0660b98426c1cba203a6fb64a61b2ece9bd576ee65b0818c0e5260ea3 |
|
MD5 | 401dcd95a910895dbf6ed9d89d6199c5 |
|
BLAKE2b-256 | 569103a2399bdff7566032155c0d8a0b59136c38253d62107f1c799c0acd96b8 |
Close
Hashes for pgpq-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 290b1ef9df4b210c8896a72b5f6bc497cab4af105d382a9825c5071d1ff92ec0 |
|
MD5 | 2f5db74b232651201102e0cbc85f232a |
|
BLAKE2b-256 | 46969091e48c5eb3fed382c456208434bfed66f9dcde943501ed069eeb2869ff |
Close
Hashes for pgpq-0.6.1-pp37-pypy37_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 51ee86a2f669fba30dcb445c2af2e671277c10d5cd15094176423b4c9a26eacb |
|
MD5 | f73a950778495df4b7de4fbedb99041f |
|
BLAKE2b-256 | a10520b0c8979b9d8e2bc7a9db65aee92b9d367dca12300985b382dbcbc6079d |
Close
Hashes for pgpq-0.6.1-cp37-abi3-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | fc758e53bb475e80885604aad9bebf6c129410c2b8a8c800270d8b025db73725 |
|
MD5 | feacf6ecf29ab545d5a55aab798083c5 |
|
BLAKE2b-256 | a92ac267750913e458b81fd3b77263791fafca8e505162e38164b40a6674e15a |
Close
Hashes for pgpq-0.6.1-cp37-abi3-musllinux_1_2_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | aff97c188953b60d068a80148db138b8f042eaf370834dbc57d6871e08288837 |
|
MD5 | 0553bc0d7dff90a0b4e293768fc2d585 |
|
BLAKE2b-256 | 7e932315c827141cb6e43fb5961f09cd1634e0488729b948c944b5ae19e4d094 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-musllinux_1_2_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | b9d1067f3936cff359dbf40bf952316369029e8d996cb98351e77fce358fb664 |
|
MD5 | c06c074fe575a5529aef537462f0ced2 |
|
BLAKE2b-256 | 2b9b20ef084770bb4d3f99931fac3da8bd81f9fb5e9ea8cd9737d5b6d9c5892d |
Close
Hashes for pgpq-0.6.1-cp37-abi3-musllinux_1_2_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | c4e30ad64831eafc89cb572375207d087b551bcc46c2daef2e54ec81a772e98d |
|
MD5 | 27f108988ea5d121e9424a3e7353453e |
|
BLAKE2b-256 | c08e392de58a73af54e645306d067cfd526734bd52ac9af06638bd129d8740f0 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-musllinux_1_2_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 2548d2572e618151230b03317478438bf9848c6ecf9bf33952e2114a53f74bd4 |
|
MD5 | 246bc08419cb4af4ed74da8bb4e7c0ce |
|
BLAKE2b-256 | e07ee193052fa711fe47d36871b6042867e3a5fcafb80107ea0f21f2544fcf7a |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | fcb63eac6437c170a66e63f71f86a5670ea57b9cb22ddd53ed05a49269f343cc |
|
MD5 | 039abe4c2230169f5872c9b66a829756 |
|
BLAKE2b-256 | fe91aa876e8a7530588e3e61f7ad1889896e82631feb52d6d46164f8bd4e50e4 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 243a7fcf1f0f35955fb914213fe7052e900d00cf0177739fe467c9d3d4ad92f8 |
|
MD5 | 04553be99010ac5b61bccd9549730d94 |
|
BLAKE2b-256 | aeb2bb3be698c6584750d29fbe7a266d0db9b63e7c29ce53f839f7032cfeacf9 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 84e82b383b837f0f9c9bbdc0245f7134abae6c9a19e046015f06d618a636c959 |
|
MD5 | 6d510fd769bf301874010a4f949e10c9 |
|
BLAKE2b-256 | f159adab719cebc30a97540f9fc0f63b1bb0a702f4dcbe264feacbe14fa480fd |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | ca65212991aae27eec13adc3fd25c9a0a3934c2cbdc4965e85a1f4e7163d8976 |
|
MD5 | 6b96a37883592a3edbb1970f7c9e494d |
|
BLAKE2b-256 | fee0196182ae92631e6cf9aea11dacc9909ad37b1f3ba678437d325787dc3427 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | cbe79b73c2ef7ad94e342a91a73b10aedbd93aac116875f703f95e74f70e197f |
|
MD5 | 69292d436042df52dc27a2208892a29f |
|
BLAKE2b-256 | 6cca74ab1864d124831f3630b6e322bb9daa96769c841f08cef4c044e45bee12 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | cca2d5f3abc64315bc45b88789b9183157cc27f09e37f26a9dfc8f38d5073cb2 |
|
MD5 | 386ec12fb0998ac9350e64d680faa6ae |
|
BLAKE2b-256 | 1cd8bb3c412a1ae117f2b4ee41a568cebe8c7486244c26f62c241ddc78d16105 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 455752d899cfb6d079749a1fdd53ea539968172b6659abe4e6e9f49d321c56af |
|
MD5 | aa122714a2d277bb56dba76b4e532106 |
|
BLAKE2b-256 | 9d9a12b2e1c058083cf32f9350fa221769adf4f714db102fedd349bf3b1dec23 |
Close
Hashes for pgpq-0.6.1-cp37-abi3-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 79f650ae69c806eb990bce6b203194a70b60c3957b7bc96965a07ca20d394151 |
|
MD5 | 837e459c3dcf7f1dd255ce26c7d938a9 |
|
BLAKE2b-256 | 25ec58567dc575fc8b4cc55057954bfe21d0588fa8f6f4a4346b64a1f7a9201b |
Close
Hashes for pgpq-0.6.1-cp37-abi3-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 12d80b06159e11e1b225d5bdf1b3ba7cfd1e32c0b1e609d6f527f3815cbcdeef |
|
MD5 | 654f2cb42f52436b4c32943e9750e5e3 |
|
BLAKE2b-256 | 7d346ae7cb2c3fb5b54de24c4331c5c4073132fae71083532c856566fdd68d8c |