Arrow -> PostgreSQL encoder
Project description
pgpq
Convert PyArrow RecordBatches to Postgres' native binary format.
Usage
import psycopg
import pyarrow.dataset as ds
from pgpq import ArrowToPostgresBinaryEncoder
dataset = ds.dataset("tests/testdata/")
encoder = ArrowToPostgresBinaryEncoder(dataset.schema)
# get the expected postgres destination schema
# note that this is _not_ the same as the incoming arrow schema
# and not necessarily the schema of your permanent table
# instead it's the schema of the data that will be sent over the wire
# which for example does not have timezones on any timestamps
pg_schema = encoder.schema()
# assemble ddl for a temporary table
# it's often a good idea to bulk load into a temp table to:
# (1) Avoid indexes
# (2) Stay in-memory as long as possible
# (3) Be more flexible with types (you can't load a SMALLINT into a BIGINT column without casting)
cols = [f"\"{col['name']}\" {col['data_type']['ddl']}" for col in pg_schema["columns"]]
ddl = f"CREATE TEMP TABLE data ({','.join(cols)})"
with psycopg.connect("postgres://postgres:postgres@localhost:5432/postgres") as conn:
with conn.cursor() as cursor:
cursor.execute(ddl) # type: ignore
with cursor.copy("COPY data FROM STDIN WITH (FORMAT BINARY)") as copy:
copy.write(encoder.write_header())
for batch in dataset.to_batches():
copy.write(encoder.write_batch(batch))
copy.write(encoder.finish())
# load into your actual table, possibly doing type casts
# cursor.execute("INSERT INTO \"table\" SELECT * FROM data")
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
pgpq-0.7.0.tar.gz
(32.3 kB
view hashes)
Built Distributions
pgpq-0.7.0-cp37-abi3-win_amd64.whl
(502.5 kB
view hashes)
pgpq-0.7.0-cp37-abi3-win32.whl
(455.5 kB
view hashes)
Close
Hashes for pgpq-0.7.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 6af46667d702ae637bf4329dc13c71d7a264db73e8684acd04d5a7dcef696866 |
|
MD5 | d91bc01fc88167f61452a8d798f04089 |
|
BLAKE2b-256 | 3ecec328a13c557ec1c508b285aad650a68605cf3d4ddfd700d8edd4017bc2bf |
Close
Hashes for pgpq-0.7.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | f68908a0675b54e1875df5c434027970ce256d30fdb8e71a03d93a556f0b9faa |
|
MD5 | b42385343a262a5a3b88aa30022ab0d4 |
|
BLAKE2b-256 | b195308c5373db2e47d50734d9d7c502525fb2ca28bd91fd35577121a9e77549 |
Close
Hashes for pgpq-0.7.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | cf3e065f48a5bbe9427ad07521e0d39f1b6f4423b0e60fcbaec5c20534374273 |
|
MD5 | f422b68fbb72d49cb4962a980fb9d953 |
|
BLAKE2b-256 | 722fcb566dd906d4e9f9b8ee7f13e8196831cbca9dc083e5f0dff60d21e70bdb |
Close
Hashes for pgpq-0.7.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 2e35ca5baebe0501602c0bcfaea8b98a846604331be0ca6b42e66ededf0344b8 |
|
MD5 | 84b3d78ad54ae57537e8cb92e88be4f5 |
|
BLAKE2b-256 | 4dce56f6c98a888f1d3cbe8aa1f839445381d5b811466a46cda153e3c1d6731d |
Close
Hashes for pgpq-0.7.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 6555e107db28aaa8e006ba00fdea495c7cfe03867bcc9c1e4a3b34155f001920 |
|
MD5 | 84d94081f99eee346da8d1e9d966c4d1 |
|
BLAKE2b-256 | 891e3f4252117168496665a32c0f0701386f854fd6601b48f7979f66797804fc |
Close
Hashes for pgpq-0.7.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | b1371a67cdff7376e729995f214a45a1ad319acb46c5de769f4eea36fd9779d9 |
|
MD5 | b550848095aeeba3c8cc5fd5fec56bc3 |
|
BLAKE2b-256 | 2919c4ba2393fb1f95bf0008d12e49c9685738db63c7e9767178241d2e933683 |
Close
Hashes for pgpq-0.7.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | c977ac41ab7ce5bacb83e431a0c28917c3f5d3a340421f703719b45b980aae9a |
|
MD5 | 11d3314e8d675401fd9942e33a97f77b |
|
BLAKE2b-256 | 1314c1cd89608a3e4bc075c6a3b3b12209b22e1576c7501e42ccf32a906b0a9f |
Close
Hashes for pgpq-0.7.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 6c9b897a763e758e0b01382917823c2fb4ed7ce214d6a5cf1f13b6c83320adb8 |
|
MD5 | b9971445786801df790ba248ddc25ff8 |
|
BLAKE2b-256 | f0063bc8824d81cd42319a82c8bc9c7e12d68c6d30c08e1a063bf796825ed9d3 |
Close
Hashes for pgpq-0.7.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | be876a52d6ea87eb8e2f739f1db0c30ab2035addc2b5d9581b05c6c07e75add8 |
|
MD5 | 05c7ee7cd16eb462d4735d910a1996de |
|
BLAKE2b-256 | c7eb989f9176b66a35169e44f50d3dcc18159f659f1917f367433d868f7328d3 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | bb193ffe8d6de84f6c325ffca4cd61dbe7b02f757b729684177597d2c680ae49 |
|
MD5 | bbe9fed84bd8533590ca9874656950f1 |
|
BLAKE2b-256 | c998be2b1bfff1799a99bafe57b38af9b8af5716e81f5ae8bbd6b14533b0e075 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-musllinux_1_2_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | ebbd11ee7cbe5c098d09629cea27a1125e28c3308ef135e41861f15f8716bd30 |
|
MD5 | 8bd5f5417fb6ab0accb4c7b77b66a0ba |
|
BLAKE2b-256 | eacb6f0847e25afcb484f6a710feb70e679eb4e2f9008988fe6d701fa3a62e53 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-musllinux_1_2_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | c91b66e0b53618c3f31b8736e5c082ee7052229333966b5f9ba6a1faf2cdb148 |
|
MD5 | 1d8d5512505c51de76e2a2530d82cfa7 |
|
BLAKE2b-256 | c49ab736bdbabca88812e887bb5bdb01468e59f460e327d209ed1cd247351ae8 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-musllinux_1_2_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 779fbe7738de911750972d5453c2c3f675e2c54d3eb673bc2b1ee433106bb671 |
|
MD5 | 9a59164fde9f33c7fd4469222c5ad447 |
|
BLAKE2b-256 | 254265c169286ef60cda03bae5b311dbdb88bba20ab34c87657cbbc0bb2675b8 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-musllinux_1_2_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 0ff5ef0e0b47180c6b04d0db93dad8e7222b62480496977d0459f78fbeb8d75a |
|
MD5 | 3bc993d2e4c7750bae9c73cfff51c608 |
|
BLAKE2b-256 | 95c157e1ed29b1929f57c5b14304fdb4f7bff5f9a2baf9277ee1daff64302295 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 8ff96d7d3b74bada724a436e2d1e82386985fdb8261f03000ca56662ccd61166 |
|
MD5 | 4bc04bfa0cfef7ef93bd65a0432e1670 |
|
BLAKE2b-256 | b604f998934a6e71b32b61715fdaf3b45032812fe5906fc1d53a55ad86829547 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 5ce0edf371075ecce990cdb91aef9686bb37658459d46cf9437909ba2c13c132 |
|
MD5 | a0bb9ef1fb82b81e49a9e5fee05f90c4 |
|
BLAKE2b-256 | dc358b13e02b15d9b5880a7474cc399fa379e4682c25e9985e5e26fda00a2f7d |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 468cc861619686d42c74e1dfdc21223e07bce7576226cba8326653153c33ca8c |
|
MD5 | cee418e9641a3a7273077c7cf7292edd |
|
BLAKE2b-256 | 6e6b746332474daebec91c9d4c13c77b41d5a6dc7a86ce21165ed7cbe17dd6a0 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 80892623707d81b0e40358326c1049f6daf8a97ae3dfec64b56f9ab6e8639933 |
|
MD5 | f2be53540ecad31f0264e6179b2cce22 |
|
BLAKE2b-256 | 9265b8953c150eb7045b27753dadcbc1b6b6e130f211d794b623856a0703d561 |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 29b278ec053389cbdc83472bd59a20bf0bf835e00af73846805b9465ab1a8115 |
|
MD5 | b166b59a142156a98e3edd0d649cded7 |
|
BLAKE2b-256 | 0116a153fac36da17c49ba95dde256052f5451a067ae9e76656b3c6a7bee6d3e |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 634fb4bfadaa9e90ebd25dc4cfcd29596c0fe0900fb4c54c89e329e2a851ac9b |
|
MD5 | 0f819ffbe345a7419510834b352c7395 |
|
BLAKE2b-256 | 5a1d568c87875d3057425f6b5714e3d7a86265e552ce29515571ef3317d078bb |
Close
Hashes for pgpq-0.7.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 16b0919ec79459d82cb299745c941c77b947bee3b0ffe97efe6227577a70402e |
|
MD5 | 3fe2a89ff25978632e4ac8c7a234cabe |
|
BLAKE2b-256 | ea45a39208efec227bf7a40561a6d28d138e6058303b6bdb2c01e00171e527fb |
Close
Hashes for pgpq-0.7.0-cp37-abi3-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 1bfbb09e262aed031fe42c521303739265ea4e0620f6b0c257c848e1b60d552a |
|
MD5 | f9fbaed5ab208ecb140a928156e68c21 |
|
BLAKE2b-256 | 09ecbec5b876ee2ba0becfa2b2e6145e5f760fe96bc6e434990c09b47ba8afcd |
Close
Hashes for pgpq-0.7.0-cp37-abi3-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 0e38e9311620c816d79e0600a543bf42be91f7e033642d28b77dcaf554c4386e |
|
MD5 | 269591937b5626e0345ea17421f4c259 |
|
BLAKE2b-256 | 7b11b975ac6b537299a102187274ff16e11bf7733b9f7f8360f70e61db8c035a |