Arrow -> PostgreSQL encoder
Project description
pgpq
Convert PyArrow RecordBatches to Postgres' native binary format.
Usage
import psycopg
import pyarrow.dataset as ds
from pgpq import ArrowToPostgresBinaryEncoder
dataset = ds.dataset("tests/testdata/")
encoder = ArrowToPostgresBinaryEncoder(dataset.schema)
# get the expected postgres destination schema
# note that this is _not_ the same as the incoming arrow schema
# and not necessarily the schema of your permanent table
# instead it's the schema of the data that will be sent over the wire
# which for example does not have timezones on any timestamps
pg_schema = encoder.schema()
# assemble ddl for a temporary table
# it's often a good idea to bulk load into a temp table to:
# (1) Avoid indexes
# (2) Stay in-memory as long as possible
# (3) Be more flexible with types (you can't load a SMALLINT into a BIGINT column without casting)
cols = [f"\"{col['name']}\" {col['data_type']['ddl']}" for col in pg_schema["columns"]]
ddl = f"CREATE TEMP TABLE data ({','.join(cols)})"
with psycopg.connect("postgres://postgres:postgres@localhost:5432/postgres") as conn:
with conn.cursor() as cursor:
cursor.execute(ddl) # type: ignore
with cursor.copy("COPY data FROM STDIN WITH (FORMAT BINARY)") as copy:
copy.write(encoder.write_header())
for batch in dataset.to_batches():
copy.write(encoder.write_batch(batch))
copy.write(encoder.finish())
# load into your actual table, possibly doing type casts
# cursor.execute("INSERT INTO \"table\" SELECT * FROM data")
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
pgpq-0.4.2.tar.gz
(38.1 MB
view hashes)
Built Distributions
pgpq-0.4.2-cp37-abi3-win_amd64.whl
(350.4 kB
view hashes)
pgpq-0.4.2-cp37-abi3-win32.whl
(325.7 kB
view hashes)
Close
Hashes for pgpq-0.4.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | a83b68179f5b46154025db4ea3d04bfe2c8722a2ece32338e86157eeb351fbdc |
|
MD5 | bbb0bee2c72e59cc8803f7210dc3fdaa |
|
BLAKE2b-256 | 9b28bc5ff3e52fce6cf370bd047a37cc772155a47a9c8507cc8bf3c9cf0c4b9d |
Close
Hashes for pgpq-0.4.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 195e7a5e2841c9deb7861e2b659f75f99e22111e8660059d905b674021ccb74e |
|
MD5 | 86754521e41fa30edfabcde87df2f368 |
|
BLAKE2b-256 | 9ec30bd0e41f3c24941ba8dde6b3f95947cc93e3a00ce805b188bcafd1d3e123 |
Close
Hashes for pgpq-0.4.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | b70709153afceddfd5b2c49be5e2b3b4ee97b825b0a0b7ac5db9a474c8deb2f0 |
|
MD5 | 570633356256b906d308390153663088 |
|
BLAKE2b-256 | b3a15f6cac9fae2c971f98b9c0f6c31759d10d2c59f88205aa1681d27e6dfbc3 |
Close
Hashes for pgpq-0.4.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 3856e86feedca36a83bc19a296c52e43d0280dc5830995e2bd80c6ef5413393b |
|
MD5 | bd152666531be3e652d6fd5ff20628f0 |
|
BLAKE2b-256 | 570b8835d4be6a54ae328d79de6d87d54affd4e1ba7e9e4936a8e91d74104bef |
Close
Hashes for pgpq-0.4.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | c516808425821559ef6389fca7c48505ed88dd3b2835563f5f0d2554f3a64408 |
|
MD5 | ad0d12647c3b336bdbf06527fa12fb28 |
|
BLAKE2b-256 | 92d029a0f0759af1922b5013e5e3b21330608ed063588b372e59372c74f7e65f |
Close
Hashes for pgpq-0.4.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 5347d314bd7f556deb435f10f9f364e1a91b49e80edbd04611f576bd314b0268 |
|
MD5 | 8c370ea41d42953f7a6a6e6a16285ba9 |
|
BLAKE2b-256 | 7335c7b7c849cc3c93c9ee24d19f48eef2530fc5f0347362dd351c3f0b9da65c |
Close
Hashes for pgpq-0.4.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 4c5ba265b86c26bd0490847ab4101b5cbe63a25513822c57e69508f369159716 |
|
MD5 | 4c7ccc4729e4a23c69a0c8d93a7bdcde |
|
BLAKE2b-256 | 6d8006918550a904ae8b85172b817cc79cae24d6ed5ce580e8dea8ab612b670b |
Close
Hashes for pgpq-0.4.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 98d9c67dc8d4ead5e87becd3a648b1350d82f7d0345207e286752b89fcec841f |
|
MD5 | fdc98e983a1868a14bb738443cdc0b48 |
|
BLAKE2b-256 | 23df710aa4d6bf22236fe5dad36689f169fd05ec6139963a6d6a9553e08d2fd4 |
Close
Hashes for pgpq-0.4.2-pp37-pypy37_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 8693862c035f59fc16d578407ed805dfb94fb3e7381eb73d80686628281921cf |
|
MD5 | c476a7ab4a4943839c865f6263714249 |
|
BLAKE2b-256 | 2a90e6aa59d5c527375b1d595556f414a42ac2dc6926cbe187303250962da1dc |
Close
Hashes for pgpq-0.4.2-cp37-abi3-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 29f1ed10e3784b528657a5b451279b47d3bfbe6daebebd140beef56fc306926f |
|
MD5 | d95dc43c2387118f0839ce993a98fe3e |
|
BLAKE2b-256 | 933fc5c790bbef119920705fb05113449069532290f0e8c2b466d6c8d479d68b |
Close
Hashes for pgpq-0.4.2-cp37-abi3-musllinux_1_2_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 6f9ec330f586d150a0438b5988e3421ecaddc14a6dec78cb02ea48e4f7294794 |
|
MD5 | 12e641e81b8ca88636ebafe551bc8a6d |
|
BLAKE2b-256 | c29da8b9c595e4dc5296d3b506adfb8900e2a2fceceb1d188b3c116488d69e85 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-musllinux_1_2_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | ffb57a3ec54d9bdad177fd2caa6d301047205f2c257cc69ac2f67d346b242231 |
|
MD5 | c5ee431df6159384cfe9fcc4cfc882a3 |
|
BLAKE2b-256 | c655e67bd831a4d9d23bb144ad01e6d0bb04f9e3b3f17463a2ee31e0575b3479 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-musllinux_1_2_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 877246182393957e8d8a96e0fa4dde1b0686b03db907079e0c23a70bb2c916cf |
|
MD5 | 5dbb8f9a10271cb9c68ef777f358cfb9 |
|
BLAKE2b-256 | 4296a9af45eb4ebef687d454004a0de56ceebd346408cabcab4594a1a1d9814c |
Close
Hashes for pgpq-0.4.2-cp37-abi3-musllinux_1_2_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | fa67665a4d68cae92877fc8c96a5718c8eff8fdb04970473b62af56820f287b4 |
|
MD5 | bdbc8cd41d332367a24d079063d906e4 |
|
BLAKE2b-256 | 0869fabfda5355aef0f22d7ff824829c0d7fda0481c8b1ad93cf9dbadf65a5b2 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 7af16edd6c0371838c9e6ffaf157f2510f6468e4603a60374dec7ee762a93195 |
|
MD5 | 476a23e6804dccdd1cba898a82914c41 |
|
BLAKE2b-256 | 8ef82cb5ff721dd380bb9b4065c97dc975b5b4243a9b8ff3e1f4a801c3148824 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 7da53ddd816497de3a5be4f41a136305c30642f9cc91913c3c810e4f2c242828 |
|
MD5 | 1a659995d492a30359861dd8ee133763 |
|
BLAKE2b-256 | 386641ac97204108b8678557b5e4cd5d02a520be3f6eae387b8ccd56d7820157 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | fb2c70c4ca0e9ef170f8560c5f0c43f5ac958bd672c333408ac1086c7f354b85 |
|
MD5 | 465d7e4465ed548ba001639d31d42bcd |
|
BLAKE2b-256 | 50e3d1a71ad5e2315a2458669d125f5ef06b0d721175e2b313711c07a49dca55 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 57a60d71deb0c859e819487b52c7e00ae1daf94c798e85b3c28820dd5d52c95d |
|
MD5 | 38e8242eb91363c3f3bc1310d8c554e6 |
|
BLAKE2b-256 | b865146698f290aafb7ad2350f65f0b4051dafb867307c03f05b9a54620646fe |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 3e0b2acfa6634e20e0ede292fe738e37f1497b1f40cd91a3fb2cbdea35b1ec17 |
|
MD5 | d23fae3f1ea2479b0163145e7d1b54ad |
|
BLAKE2b-256 | 1fcd4567b3bed2c0a297edae4a9553ef8130af8c9ab73614eaef45c18ac08355 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 4fed2aa6df3c140f391eefc6a1f076b099b22af2a883ff7df1bed69176ebe488 |
|
MD5 | bc6848794a1e0a3548bddcf56985c114 |
|
BLAKE2b-256 | c0d4aaf4e195201f3ec10660bdf0d56f06d5b4a5aa3cb0cebe3720983b6713e2 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 86488dff197b7b2b6027fbb6df3e86c86920bfa248765a88cd1e5a6228c338c6 |
|
MD5 | d72b5ab8b4e67fc7c4e37177ba8cff35 |
|
BLAKE2b-256 | 74cd2997e9fde8da5dc80c4f2b63e0f80e24634338455bc2be4a515a1f9f3a9e |
Close
Hashes for pgpq-0.4.2-cp37-abi3-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 93ec3c7689edf3eb437ff0ebeea076bf47a5618e8dbce5774a736524f546e17e |
|
MD5 | abb8d8daf7ce1c9c2a045e92883b1cfa |
|
BLAKE2b-256 | 2a76b347eb83f1901485a3b049b51c6ca13c39f48f487b05b7dda1c5f7917d49 |
Close
Hashes for pgpq-0.4.2-cp37-abi3-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 48391decf0fc5638f3d510e464c0271b36145fb657476494accc9786914515ee |
|
MD5 | 324d3a8277dc54d1c94e2c245c47efde |
|
BLAKE2b-256 | 412002b55b5760562502c9b679ead5194f78def52cc41155bcbc7e48c94beac0 |