Arrow -> PostgreSQL encoder
Project description
pgpq
Convert PyArrow RecordBatches to Postgres' native binary format.
Usage
import psycopg
import pyarrow.dataset as ds
from pgpq import ArrowToPostgresBinaryEncoder
dataset = ds.dataset("tests/testdata/")
encoder = ArrowToPostgresBinaryEncoder(dataset.schema)
# get the expected postgres destination schema
# note that this is _not_ the same as the incoming arrow schema
# and not necessarily the schema of your permanent table
# instead it's the schema of the data that will be sent over the wire
# which for example does not have timezones on any timestamps
pg_schema = encoder.schema()
# assemble ddl for a temporary table
# it's often a good idea to bulk load into a temp table to:
# (1) Avoid indexes
# (2) Stay in-memory as long as possible
# (3) Be more flexible with types (you can't load a SMALLINT into a BIGINT column without casting)
cols = [f"\"{col['name']}\" {col['data_type']['ddl']}" for col in pg_schema["columns"]]
ddl = f"CREATE TEMP TABLE data ({','.join(cols)})"
with psycopg.connect("postgres://postgres:postgres@localhost:5432/postgres") as conn:
with conn.cursor() as cursor:
cursor.execute(ddl) # type: ignore
with cursor.copy("COPY data FROM STDIN WITH (FORMAT BINARY)") as copy:
copy.write(encoder.write_header())
for batch in dataset.to_batches():
copy.write(encoder.write_batch(batch))
copy.write(encoder.finish())
# load into your actual table, possibly doing type casts
# cursor.execute("INSERT INTO \"table\" SELECT * FROM data")
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
pgpq-0.5.0.tar.gz
(24.3 kB
view hashes)
Built Distributions
pgpq-0.5.0-cp37-abi3-win_amd64.whl
(349.8 kB
view hashes)
pgpq-0.5.0-cp37-abi3-win32.whl
(325.6 kB
view hashes)
Close
Hashes for pgpq-0.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 29748cb2659bcb788d4cfb161f6497dbc98897a9f6d68d00e648ba7b783f7885 |
|
MD5 | e6cb2a3c48a34fc24f0b01bc0c0cf2c3 |
|
BLAKE2b-256 | d8f812867a9445bb21e02c3a5bb8a4018f9efce94516cd7284269dd8c5b21328 |
Close
Hashes for pgpq-0.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 7dffc2bd18c57aa62641903d37a7a456aecd633e03ae7967f478bdaad60cdf2f |
|
MD5 | 344ae253311133ebb57dad5b8c827892 |
|
BLAKE2b-256 | 7ff09fb69a17d66c16ac7fde5f7e1dfeff8d458ac31b6bab39b813bb1d78d166 |
Close
Hashes for pgpq-0.5.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 1a732c933c871edfe4ee2dc706c90f8b656feeb0fb3d385c266a124df15f1dd7 |
|
MD5 | af25011996dceb9dafb9cf0d5c0a979d |
|
BLAKE2b-256 | 48647f24184598a37ae104c5f0184352a41726328c9a1e229da748b7ef2e33f8 |
Close
Hashes for pgpq-0.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 61b94997c371bbb698c0e57dd89f88d8c0851f7eabba164a9b72b9bc2b8e6226 |
|
MD5 | fa28225faba8b3eb6d9c1d6073a718f7 |
|
BLAKE2b-256 | 7641872bc74fb2a1ee35624fd84dc686a4619a3da9fc8c5e043e1cf4f08ca45d |
Close
Hashes for pgpq-0.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | a16803bf17ee1128a1b95cada69f8b9e715b1848bd88d754a13748b7585089e2 |
|
MD5 | 4cf94f1562ace6a2d40c811be29ddcdd |
|
BLAKE2b-256 | f5fe10b3ac86c26fccf9427b9fe1e8d516579d7db85c9a06842010c548a46d65 |
Close
Hashes for pgpq-0.5.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | f1c91094e7284ea1b51522dd81c5998e29268d685ee152d3dd93d3fbd6d02552 |
|
MD5 | 39156b749deee4cd42cb9bfa820e9e81 |
|
BLAKE2b-256 | 8a91836b084f27e72aa544d40644af70a9bc2a108efa0dc9b89767be64f73194 |
Close
Hashes for pgpq-0.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 58e0ef2496851c55786786d83b36efe166b2f8eec4af9dee59f4f42ce9c6af96 |
|
MD5 | 266b184f2121b5d10c37a97376858879 |
|
BLAKE2b-256 | 3252b603c169515cc1ffe681f4fcfdf9d0bbfe771e8342036f6f0213360645da |
Close
Hashes for pgpq-0.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | c7db8770dcb740f092962e9fc880513a0ec5bd9d8ebb191bf0a7544abc837c32 |
|
MD5 | a43b7cb259317ace48c52b10c3aadd5e |
|
BLAKE2b-256 | 7e86e83a0b279ffedc3f90880c9637880bc2a7f23a6b37fe17bb6463088be183 |
Close
Hashes for pgpq-0.5.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 046857c8c92453b99646db9843e9dafec4f63408d8bf9acf443b0398e148cb33 |
|
MD5 | 42ec21e6b268172497288b7021a2d745 |
|
BLAKE2b-256 | b3f4b838b09873571d530b35d7c98950400b4791dca58466a76aaa3ed3dc2a59 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-win_amd64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | a1b7efbd71eea06634bb008ef2889d756fa4c69314d4e7356299e7445ff08294 |
|
MD5 | aecf30b1a436436dedc2fe29c09638b0 |
|
BLAKE2b-256 | 06e3f05635e8ccebf4815ece131fe38450cf304c0d40d3065ac54931865f1725 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-musllinux_1_2_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 5008db2b4a7b4f8230d0218e53133e720efc3cdba4ebf6b311b78a98bf7a2c5a |
|
MD5 | ddac561e16df8e78275f4193fbfb971f |
|
BLAKE2b-256 | 1055eaebc1b05801ae53e23b27419edbda55781ceac945d07ac72b0e22af7289 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-musllinux_1_2_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | ef3ac6537e434f310e2c9c5b04ed5044799066c3beebc475d7a186e0551fb16c |
|
MD5 | cf313f3b130b426aadcb9b59efda3dc3 |
|
BLAKE2b-256 | f338ecd34ebb60b251bb54cf16b7c3890f8d24f83b1559ec7c1b8ab9da3d3c8b |
Close
Hashes for pgpq-0.5.0-cp37-abi3-musllinux_1_2_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | ee311ba5d3dc4c8c08573d7c639ba9bd9fdb9eb1671f4829e9e4af3b676fcd56 |
|
MD5 | 0753cf15fb02c94b74415da90c51ef6d |
|
BLAKE2b-256 | bc77fefe7325a296197132ce40824f54216f564897170949d6a20410ec43b106 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-musllinux_1_2_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 4bcd61e45d5b59fd35a158a62da6c10860d81e690717a8e1ad83950ae09f5f66 |
|
MD5 | 832efb92e521fbf1fad06d1a546f1549 |
|
BLAKE2b-256 | 55510b006082dc4e5eacc7d73637b8fd2b0999025c453430d7964a258c6f70e3 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 2b6e4e91ebf96d64a78e6c084f4c9046df5e410350109794355204f85cfdecc4 |
|
MD5 | 9fb8889444003daf46bd51878354b00b |
|
BLAKE2b-256 | bacd67bd1fc5677dd88d3c4b0ca2fe45d491f65539ee5e86b08ca6e69e09d6ac |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | e7f557497a8125a7e15942038beba634ddb14b86d8371b1795f45509337d72eb |
|
MD5 | f55383ff4153dfdcff76e59fca5c1b1f |
|
BLAKE2b-256 | d6f5442362a757490f3844b892d13fa2d278ffeb0e3669ae8e020f16f87af3eb |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 01131f9d877d300c45af28741860680238f6b53e813ae331b24be4f05b597565 |
|
MD5 | cf04a05f48c9106d2100bff84d689bf3 |
|
BLAKE2b-256 | 5f7785630b5e689c086d63c2ac1ee4545377680f6263e809f3c79e0b19b3a648 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 1246c38b0b9eccd947104e910d2552d842b00dd33b574eaa620924e41283ef4a |
|
MD5 | b57fa3cc0811939bf6f601a2e59096bc |
|
BLAKE2b-256 | 2863c48a7c59395fa9156c4708860f71960a7c222f8644eb5620e81cefc1453d |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 22e54bd12e7ab807f68076fb2632a7503a109753333565c7d3dfa2ce125580b7 |
|
MD5 | be55d197a5b5283453a2f3628e41771c |
|
BLAKE2b-256 | 872d24e42d9fae32cb41ee212c990108feb5a2d117788090e467b54f64bc944b |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 5415f8092f4cdd8905153001c2297b37cc21c52085ab9c9ee5e0f26cc098eaad |
|
MD5 | 0f6e438cf76b7e618ec45e8bfce69521 |
|
BLAKE2b-256 | 6e14f635a6266d7057675aba0bb7dd30de0d92a537827cf849f966e7e143b62b |
Close
Hashes for pgpq-0.5.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 513f76c19d0bd9ae41e675a81094e5bdfae6f8ccabf08af7084fa95f6547e046 |
|
MD5 | 11858fad8f7b0c751b4bd200cd0d5ab4 |
|
BLAKE2b-256 | 61c050b369ed8ed77f9a78b9ebfdbbd81993272fb23cf566de6fc4882bfa9172 |
Close
Hashes for pgpq-0.5.0-cp37-abi3-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | d45956a937958d401978c80a181a2721a4f8daedd60dff165d993d270f16bc42 |
|
MD5 | ab8031ebb06a825bf3c8c15524f79a50 |
|
BLAKE2b-256 | 7e9cba174467f2d1c9be23eb7744253eb6a82d6a2ab537bf0d79981efbe95aea |
Close
Hashes for pgpq-0.5.0-cp37-abi3-macosx_10_7_x86_64.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | 53735c2c3326883b5a585dc33f64cec93bdf9d96a1c4708e8c8eb7eea906341e |
|
MD5 | 3329fdae77527deb29cbba086762031e |
|
BLAKE2b-256 | 74202d092e7c54a9252bbcb2b87f5cd3cf924ec21f4db3ba3a3094cfca17e9df |