pax_global_header00006660000000000000000000000064150451322030014505gustar00rootroot0000000000000052 comment=865540f5979592542be904ce2f754fc9244a0e6e pg_rewrite-REL2_0_0/000077500000000000000000000000001504513220300143605ustar00rootroot00000000000000pg_rewrite-REL2_0_0/.dir-locals.el000066400000000000000000000002451504513220300170120ustar00rootroot00000000000000((c-mode . ((c-basic-offset . 4) (c-file-style . "bsd") (fill-column . 78) (indent-tabs-mode . t) (tab-width . 4)))) pg_rewrite-REL2_0_0/.github/000077500000000000000000000000001504513220300157205ustar00rootroot00000000000000pg_rewrite-REL2_0_0/.github/workflows/000077500000000000000000000000001504513220300177555ustar00rootroot00000000000000pg_rewrite-REL2_0_0/.github/workflows/regression.yml000066400000000000000000000017451504513220300226670ustar00rootroot00000000000000name: Build on: [push, pull_request] jobs: build: runs-on: ubuntu-latest defaults: run: shell: sh strategy: matrix: pgversion: - 17 env: PGVERSION: ${{ matrix.pgversion }} steps: - name: checkout uses: actions/checkout@v3 - name: install pg run: | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -v $PGVERSION -p -i sudo -u postgres createuser -s "$USER" - name: build run: | make PROFILE="-Werror" sudo -E make install - name: test run: | sudo pg_conftool set shared_preload_libraries pg_rewrite sudo pg_conftool set wal_level logical sudo pg_conftool set max_replication_slots 1 sudo pg_ctlcluster $PGVERSION main restart make installcheck - name: show regression diffs if: ${{ failure() }} run: | cat /home/runner/work/pg_rewrite/pg_rewrite/output_iso/regression.diffs pg_rewrite-REL2_0_0/.gitignore000066400000000000000000000000501504513220300163430ustar00rootroot00000000000000*~ *.o *.so results/ GPATH GRTAGS GTAGS pg_rewrite-REL2_0_0/LICENSE000066400000000000000000000020471504513220300153700ustar00rootroot00000000000000Copyright (c) 2021-2023, Cybertec PostgreSQL International GmbH Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL Cybertec PostgreSQL International GmbH BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF Cybertec PostgreSQL International GmbH HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Cybertec PostgreSQL International GmbH SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND Cybertec PostgreSQL International GmbH HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. pg_rewrite-REL2_0_0/Makefile000066400000000000000000000010221504513220300160130ustar00rootroot00000000000000PG_CONFIG ?= pg_config MODULE_big = pg_rewrite OBJS = pg_rewrite.o concurrent.o $(WIN32RES) PGFILEDESC = "pg_rewrite - tools for maintenance that requires table rewriting." EXTENSION = pg_rewrite DATA = pg_rewrite--1.0.sql pg_rewrite--1.0--1.1.sql pg_rewrite--1.1--1.2.sql\ pg_rewrite--1.2--2.0.sql pg_rewrite--1.3--2.0.sql DOCS = pg_rewrite.md REGRESS = pg_rewrite generated #ISOLATION = pg_rewrite_concurrent pg_rewrite_concurrent_partition \ pg_rewrite_concurrent_toast PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) pg_rewrite-REL2_0_0/NEWS000066400000000000000000000023041504513220300150560ustar00rootroot00000000000000Release 2.0 =========== 1. This release makes the extension useful in more use cases. Besides turning a non-partitioned table into a partitioned one, it can be used to change 1) data type of column(s), 2) order of columns, 3) tablespace. 2. A single function `rewrite_table()` is used now to handle all the use cases. 3. Constraints are handled in a more convenient way. The extension now takes care of creating the constraints on the target table according to the source table. The user only needs to validate the constrants after the rewriting has finished. Unlike the previous release, the rewritten table can be referenced by foreign key constraints. Note: The `rewrite.check_constraints` configuration variable was removed. If there is a risk that other users could run `ALTER TABLE` on the table during rewriting, please revoke the corresponding privileges from them temporarily. Release 1.1.1 ============= This release only adjusts the code so it is compatible with PostgreSQL server version 17. Release 1.1.0 ============= New Features ------------ 1. Make the code compatible with PostgreSQL server version 16. 2. Added progress monitoring.pg_rewrite-REL2_0_0/README.md000066400000000000000000000263461504513220300156520ustar00rootroot00000000000000# pg_rewrite `pg_rewrite` is a tool to rewrite table (i.e. to copy its data to a new file). It allows both read and write access to the table during the rewriting. Following are the most common reasons to rewrite a table: 1. Change data type of column(s) Typically this is needed if the existing data type is running out of values. For example, you may need to change `interger` type to `bigint`. `ALTER TABLE` command can do that too, but it allows neither write nor read access to the table during the rewriting. 2. Partition the table If you realize that your table is getting much bigger than expected and that partitioning would make your life easier, the next question may be how to copy the existing data to the new, partitioned table without stopping all the applications that run DML commands on the table. (When you decide to use partitioning, the amount of data to copy might already be significant, so the copying might need a while.) 3. Change order of columns If you conclude that a different order of columns would save significant disk space (due to reduced paddding), the problem boils down to copying data to a new table like in 2). Again, you may need `pg_rewrite` to make the change smooth. 4. Move table into another tablespace. `ALTER TABLE` command can do that, but it allows neither write nor read access to the table during the rewriting. With `pg_rewrite`, you only need to create the new table in the desired tablespace. The rest is identical to the other use cases. Note that the following use cases can be combined in a single rewrited. # INSTALLATION Install PostgreSQL before proceeding. Make sure to have `pg_config` binary, these are typically included in `-dev` and `-devel` packages. PostgreSQL server version 13 or later is required. ```bash git clone https://github.com/cybertec-postgresql/pg_rewrite.git cd pg_rewrite git checkout make make install ``` Add these to `postgresql.conf`: ``` wal_level = logical max_replication_slots = 1 # ... or add 1 to the current value. shared_preload_libraries = 'pg_rewrite' # ... or add the library to the existing ones. ``` Restart the cluster, and invoke: ``` CREATE EXTENSION pg_rewrite; ``` # USAGE Assume you have a table defined like this ``` CREATE TABLE measurement ( id int, city_id int not null, logdate date not null, peaktemp int, PRIMARY KEY(id, logdate) ); ``` and you need to replace it with a partitioned table. At the same time, you want to change the data type of the `id` column to `bigint`. ``` CREATE TABLE measurement_aux ( id bigint, city_id int not null, logdate date not null, peaktemp int, PRIMARY KEY(id, logdate) ) PARTITION BY RANGE (logdate); ``` Then create partitions for all the rows currently present in the `measurement` table, and also for the data that might be inserted during processing: ``` CREATE TABLE measurement_y2006m02 PARTITION OF measurement_aux FOR VALUES FROM ('2006-02-01') TO ('2006-03-01'); CREATE TABLE measurement_y2006m03 PARTITION OF measurement_aux FOR VALUES FROM ('2006-03-01') TO ('2006-04-01'); -- ... ``` *It's essential that both the source (`measurement`) and target (`measurement_aux`) table have an identity index. It is needed to process data changes that applications make while data is being copied from the source to the target table. If the replica identity of the table is DEFAULT or FULL, primary key constraint provides the identity index. If your table has no primary key, you need to set the identity index explicitly using the [ALTER COMMAND ... REPLICA IDENTITY USING INDEX ...][1] command. Also note that the key (i.e. column list) of the identity index of the source and target table must be identical.* Then, in order to copy the data into the target table, run the `rewrite_table()` function and pass it both the source and target table, as well as a new table name for the source table. For example: ``` SELECT rewrite_table('measurement', 'measurement_aux', 'measurement_old'); ``` The call will first copy all rows from `measurement` to `measurement_aux`. Then it will apply to `measurement_aux` all the data changes (INSERT, UPDATE, DELETE) that took place in `measurement` during the copying. Next, it will lock `measurement` so that neither read nor write access is possible. Finally it will rename `measurement` to `measurement_old` and `measurement_aux` to `measurement`. Thus `measurement` ends up to be the partitioned table, while `measurement_old` is the original, non-partitioned table. If a column of the target table has a different data type from the corresponding column of the source table, an implicit or assignment cast must exist between the two types. # Constraints The target table should obviously end up with the same constraints as the source table. It's recommended to handle constraints creation this way: 1. Add PRIMARY KEY, UNIQUE and EXCLUDE constraints of the source table to the target table before you call `rewrite_table()`. These are enforced during the rewriting, so any violation would make `rewrite_table()` fail (ROLLBACK). (The constraints must have been enforced in the source table, but it does not hurt to check them in the target table, especially if the column data type is being changed.) 2. If the version of PostgreSQL server is 17 or lower, add NOT NULL constraints of the source table to the target table. `rewrite_table()` by-passes validation of these, but all the rows it inserts into the target table must have been validated in the source table. Even if the column data tape is different in the target table, the data type conversion should not turn non-NULL value to NULL or vice versa. 3. CHECK constraints are created automatically by `rewrite_table()` (according to the source table) when all the data changes have been applied to the target table. However, these constraints are created as NOT VALID, so you need to use the `ALTER TABLE ... VALIDATE CONSTRAINT ...` command to validate them. (The function does not create these constraints immediately as valid, because that could imply blocking access to the table for significant time.) 4. If the version of PostgreSQL server is 18 or higher, NOT NULL constraints are also created automatically and need to be validated using the `ALTER TABLE ... VALIDATE CONSTRAINT ...` command. 5. FOREIGN KEY constraints are also created automatically (according to the source table) and need to be validated using the `ALTER TABLE ... VALIDATE CONSTRAINT ...` command, unless the referencing table is partitioned and the version of PostgreSQL server is 17 or lower: those versions do not support the NOT VALID option for partitioned tables. Therefore, if the referencing table is partitioned and if the server version is 17 or lower, you need to use the `ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY ...` command after `rewrite_table()` has finished. Please run the command as soon as possible to minimize the risk that applications modify the database in a way that violates the constraints. 6. Drop all foreign keys involving the source table. You probably want to drop the source table anyway, but if you don't, you should at least drop its FOREIGN KEY constraints. As the table was renamed, applications will no longer update it. Therefore, attempts to update the other tables involved in its foreign keys may cause errors. # Sequences If a sequence is used to generate column value in the source table (typically the column data type is `serial` or the column is declared `GENARATED ... AS IDENTITY`), and if `rewrite_table()` finds the corresponding sequence for the target table, it sets its value according to the sequence for the source table. If it cannot identify the sequence for the target table, a log message is printed out. # Progress monitoring If `rewrite_table()` takes long time to finish, you might be interested in the progress. The `pg_rewrite_progress` view shows all the pending calls of the function in the current database. The `src_table`, `dst_table` and `src_table_new` columns contain the arguments of the `rewrite_table()` function. `ins_initial` is the number of tuples inserted into the new table storage during the "initial load stage", i.e. the number of tuples present in the table before the processing started. On the other hand, `ins`, `upd` and `del` are the numbers of tuples inserted, updated and deleted by applications during the table processing. (These "concurrent data changes" must also be incorporated into the partitioned table, otherwise they'd get lost.) # Limitations 1. If the target table is partitioned, it's not allowed to have foreign tables as partitions. 2. Indexes are not renamed. While the target table (`measurement_aux` above) is renamed to the source table (`measurement`), its indexes are not renamed to match the source table. If you consider it a problem, please use the `ALTER INDEX` command to rename them. This operation blocks neither reads nor writes. # Configuration Following is the description of the configuration variables that affect behavior of the functions of this extension. * `rewrite.max_xlock_time` Although the table being processed is available for both read and write operations by other transactions most of the time, an exclusive lock is needed to finalize the processing (i.e. to do the table renaming), which blocks both read and write access. This should take very short time that users should harly notice. However, if a significant amount of changes took place in the source table while the extension was waiting for the (exclusive) lock, the outage might take proportionally longer time. The point is that those changes need to be propagated to the target table before the exclusive lock can be released. If the extension function seems to block access to tables too much, consider setting `rewrite.max_xlock_time` GUC parameter. For example: ``` SET rewrite.max_xlock_time TO 100; ``` Tells that the exclusive lock shouldn't be held for more than 0.1 second (100 milliseconds). If more time is needed for the final stage, the particular function releases the exclusive lock, processes the changes committed by the other transactions in between and tries the final stage again. Error is reported if the lock duration is exceeded a few more times. If that happens, you should either increase the setting or try to process the problematic table later, when the write activity is lower. The default value is `0`, meaning that the final stage can take as much time as it needs. # Concurrency 1. While the rewrite_table() function is executing, `ALTER TABLE` command on the same table should be blocked until the rewriting is done. However, in some cases the `ALTER TABLE` command and the rewrite_table() function might end up in a deadlock. Therefore it's recommended not to run ALTER TABLE on a table which is being rewritten. 2. The `rewrite_table()` function allows for MVCC-unsafe behavior described in the first paragraph of [mvcc-caveats][2]. [1] https://www.postgresql.org/docs/17/sql-altertable.html [2] https://www.postgresql.org/docs/current/mvcc-caveats.html pg_rewrite-REL2_0_0/concurrent.c000066400000000000000000000710441504513220300167140ustar00rootroot00000000000000/*----------------------------------------------------------------------------------- * * concurrent.c * Tools for maintenance that requires table rewriting. * * This file handles changes that took place while the data is being * copied from one table to another one. * * Copyright (c) 2021-2025, Cybertec PostgreSQL International GmbH * *----------------------------------------------------------------------------------- */ #include "pg_rewrite.h" #include "access/heaptoast.h" #include "executor/execPartition.h" #include "executor/executor.h" #include "replication/decode.h" #include "utils/rel.h" typedef enum { CHANGE_INSERT, CHANGE_UPDATE_OLD, CHANGE_UPDATE_NEW, CHANGE_DELETE } ConcurrentChangeKind; typedef struct ConcurrentChange { /* See the enum above. */ ConcurrentChangeKind kind; /* * The actual tuple. * * The tuple data follows the ConcurrentChange structure. Before use make * sure the tuple is correctly aligned (ConcurrentChange can be stored as * bytea) and that tuple->t_data is fixed. */ HeapTupleData tup_data; } ConcurrentChange; static void apply_concurrent_changes(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, DecodingOutputState *dstate, ScanKey key, int nkeys, Relation ident_index, TupleTableSlot *slot_dst_ind, partitions_hash *partitions, TupleConversionMapExt *conv_map, struct timeval *must_complete); static void apply_insert(HeapTuple tup, TupleTableSlot *slot, EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, partitions_hash *partitions, TupleConversionMapExt *conv_map, BulkInsertState bistate); static void apply_update_or_delete(HeapTuple tup, HeapTuple tup_old, ConcurrentChangeKind change_kind, EState *estate, ScanKey key, int nkeys, Relation ident_index, TupleTableSlot *slot_dst, TupleTableSlot *slot_dst_ind, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, partitions_hash *partitions, TupleConversionMapExt *conv_map); static void find_tuple_in_partition(HeapTuple tup, Relation partition, partitions_hash *partitions, ScanKey key, int nkeys, ItemPointer ctid); static void find_tuple(HeapTuple tup, Relation rel, Relation ident_index, ScanKey key, int nkeys, ItemPointer ctid, TupleTableSlot *slot_dst_ind); static bool processing_time_elapsed(struct timeval *utmost); static void plugin_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, bool is_init); static void plugin_shutdown(LogicalDecodingContext *ctx); static void plugin_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn); static void plugin_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, XLogRecPtr commit_lsn); static void plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change); static void store_change(LogicalDecodingContext *ctx, ConcurrentChangeKind kind, HeapTuple tuple); static HeapTuple get_changed_tuple(ConcurrentChange *change); static bool plugin_filter(LogicalDecodingContext *ctx, RepOriginId origin_id); /* * Decode and apply concurrent changes. If there are too many of them, split * the processing into multiple iterations so that the intermediate storage * (tuplestore) is not likely to be written to disk. * * See check_catalog_changes() for explanation of lock_held argument. * * Returns true if must_complete is NULL or if managed to complete by the time * *must_complete indicates. */ bool pg_rewrite_process_concurrent_changes(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, LogicalDecodingContext *ctx, XLogRecPtr end_of_wal, ScanKey ident_key, int ident_key_nentries, Relation ident_index, TupleTableSlot *slot_dst_ind, LOCKMODE lock_held, partitions_hash *partitions, TupleConversionMapExt *conv_map, struct timeval *must_complete) { DecodingOutputState *dstate; bool done; /* * Some arguments are specific to partitioned table, some to * non-partitioned one. XXX Is some refactoring needed here, such as using * an union? */ Assert((ident_index && slot_dst_ind && partitions == NULL && proute == NULL) || (ident_index == NULL && slot_dst_ind == NULL&& partitions && proute)); dstate = (DecodingOutputState *) ctx->output_writer_private; /* * If some changes could not be applied due to time constraint, make sure * the tuplestore is empty before we insert new tuples into it. */ if (dstate->nchanges > 0) apply_concurrent_changes(estate, mtstate, proute, dstate, ident_key, ident_key_nentries, ident_index, slot_dst_ind, partitions, conv_map, must_complete); /* Ran out of time? */ if (dstate->nchanges > 0) return false; done = false; while (!done) { pg_rewrite_exit_if_requested(); done = pg_rewrite_decode_concurrent_changes(ctx, end_of_wal, must_complete); if (processing_time_elapsed(must_complete)) /* Caller is responsible for applying the changes. */ return false; if (dstate->nchanges == 0) continue; /* * XXX Consider if it's possible to check *must_complete and stop * processing partway through. Partial cleanup of the tuplestore seems * non-trivial. */ apply_concurrent_changes(estate, mtstate, proute, dstate, ident_key, ident_key_nentries, ident_index, slot_dst_ind, partitions, conv_map, must_complete); /* Ran out of time? */ if (dstate->nchanges > 0) return false; } return true; } /* * Decode logical changes from the XLOG sequence up to end_of_wal. * * Returns true iff done (for now), i.e. no more changes below the end_of_wal * can be decoded. */ bool pg_rewrite_decode_concurrent_changes(LogicalDecodingContext *ctx, XLogRecPtr end_of_wal, struct timeval *must_complete) { DecodingOutputState *dstate; ResourceOwner resowner_old; /* * Invalidate the "present" cache before moving to "(recent) history". * * Note: The cache entry of the transient relation is not affected * (because it was created by the current transaction), but the tuple * descriptor shouldn't change anyway (as opposed to index info, which we * change at some point). Moreover, tuples of the transient relation * should not actually be deconstructed: reorderbuffer.c records the * tuples, but - as it never receives the corresponding commit record - * does not examine them in detail. */ InvalidateSystemCaches(); dstate = (DecodingOutputState *) ctx->output_writer_private; resowner_old = CurrentResourceOwner; CurrentResourceOwner = dstate->resowner; PG_TRY(); { while (ctx->reader->EndRecPtr < end_of_wal) { XLogRecord *record; XLogSegNo segno_new; char *errm = NULL; XLogRecPtr end_lsn; record = XLogReadRecord(ctx->reader, &errm); if (errm) elog(ERROR, "%s", errm); if (record != NULL) LogicalDecodingProcessRecord(ctx, ctx->reader); if (processing_time_elapsed(must_complete)) break; /* * If WAL segment boundary has been crossed, inform PG core that * we no longer need the previous segment. */ end_lsn = ctx->reader->EndRecPtr; XLByteToSeg(end_lsn, segno_new, wal_segment_size); if (segno_new != rewrite_current_segment) { LogicalConfirmReceivedLocation(end_lsn); elog(DEBUG1, "pg_rewrite: confirmed receive location %X/%X", (uint32) (end_lsn >> 32), (uint32) end_lsn); rewrite_current_segment = segno_new; } pg_rewrite_exit_if_requested(); } InvalidateSystemCaches(); CurrentResourceOwner = resowner_old; } PG_CATCH(); { InvalidateSystemCaches(); CurrentResourceOwner = resowner_old; PG_RE_THROW(); } PG_END_TRY(); elog(DEBUG1, "pg_rewrite: %.0f changes decoded but not applied yet", dstate->nchanges); return ctx->reader->EndRecPtr >= end_of_wal; } /* * Apply changes that happened during the initial load. * * Scan key is passed by caller, so it does not have to be constructed * multiple times. Key entries have all fields initialized, except for * sk_argument. */ static void apply_concurrent_changes(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, DecodingOutputState *dstate, ScanKey key, int nkeys, Relation ident_index, TupleTableSlot *slot_dst_ind, partitions_hash *partitions, TupleConversionMapExt *conv_map, struct timeval *must_complete) { BulkInsertState bistate = NULL; HeapTuple tup_old = NULL; Relation rel_dst; TupleTableSlot *slot_dst; if (dstate->nchanges == 0) return; /* See perform_initial_load() */ if (proute == NULL) bistate = GetBulkInsertState(); /* * Slot for the destination relation is needed even in the partitioned * case, to route changes to partitions. */ rel_dst = mtstate->resultRelInfo->ri_RelationDesc; slot_dst = MakeSingleTupleTableSlot(RelationGetDescr(rel_dst), &TTSOpsHeapTuple); /* * In case functions in the index need the active snapshot and caller * hasn't set one. */ PushActiveSnapshot(GetTransactionSnapshot()); while (tuplestore_gettupleslot(dstate->tstore, true, false, dstate->tsslot)) { bool shouldFree; HeapTuple tup_change, tup; char *change_raw; ConcurrentChange *change; bool isnull[1]; Datum values[1]; Assert(dstate->nchanges > 0); dstate->nchanges--; /* Get the change from the single-column tuple. */ tup_change = ExecFetchSlotHeapTuple(dstate->tsslot, false, &shouldFree); heap_deform_tuple(tup_change, dstate->tupdesc_change, values, isnull); Assert(!isnull[0]); /* This is bytea, but char* is easier to work with. */ change_raw = (char *) DatumGetByteaP(values[0]); change = (ConcurrentChange *) VARDATA(change_raw); tup = get_changed_tuple(change); if (change->kind == CHANGE_UPDATE_OLD) { Assert(tup_old == NULL); tup_old = tup; } else if (change->kind == CHANGE_INSERT) { Assert(tup_old == NULL); apply_insert(tup, slot_dst, estate, mtstate, proute, partitions, conv_map, bistate); } else if (change->kind == CHANGE_UPDATE_NEW || change->kind == CHANGE_DELETE) { apply_update_or_delete(tup, tup_old, change->kind, estate, key, nkeys, ident_index, slot_dst, slot_dst_ind, mtstate, proute, partitions, conv_map); /* The function is responsible for freeing. */ if (tup_old != NULL) tup_old = NULL; } else elog(ERROR, "Unrecognized kind of change: %d", change->kind); /* If there's any change, make it visible to the next iteration. */ if (change->kind != CHANGE_UPDATE_OLD) { CommandCounterIncrement(); UpdateActiveSnapshotCommandId(); } /* TTSOpsMinimalTuple has .get_heap_tuple==NULL. */ Assert(shouldFree); pfree(tup_change); /* * If there is a limit on the time of completion, check it * now. However, make sure the loop does not break if tup_old was set * in the previous iteration. In such a case we could not resume the * processing in the next call. */ if (must_complete && tup_old == NULL && processing_time_elapsed(must_complete)) /* The next call will process the remaining changes. */ break; } /* If we could not apply all the changes, the next call will do. */ if (dstate->nchanges == 0) tuplestore_clear(dstate->tstore); PopActiveSnapshot(); /* Cleanup. */ if (bistate) FreeBulkInsertState(bistate); ExecDropSingleTupleTableSlot(slot_dst); } static void apply_insert(HeapTuple tup, TupleTableSlot *slot, EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, partitions_hash *partitions, TupleConversionMapExt *conv_map, BulkInsertState bistate) { List *recheck; Relation rel_ins; ResultRelInfo *rri = NULL; if (conv_map) tup = convert_tuple_for_dest_table(tup, conv_map); ExecStoreHeapTuple(tup, slot, false); if (proute) { PartitionEntry *entry; /* Which partition does the tuple belong to? */ rri = ExecFindPartition(mtstate, mtstate->rootResultRelInfo, proute, slot, estate); rel_ins = rri->ri_RelationDesc; entry = get_partition_entry(partitions, RelationGetRelid(rel_ins)); bistate = entry->bistate; /* * Make sure the tuple matches the partition. The typical problem we * address here is that a partition was attached that has a different * order of columns. */ if (entry->conv_map) { tup = convert_tuple_for_dest_table(tup, entry->conv_map); ExecClearTuple(slot); ExecStoreHeapTuple(tup, slot, false); } } else { /* Non-partitioned table. */ rri = mtstate->resultRelInfo; rel_ins = rri->ri_RelationDesc; /* Use bistate passed by the caller. */ } Assert(bistate != NULL); table_tuple_insert(rel_ins, slot, GetCurrentCommandId(true), 0, bistate); #if PG_VERSION_NUM < 140000 estate->es_result_relation_info = rri; #endif /* Update indexes. */ recheck = ExecInsertIndexTuples( #if PG_VERSION_NUM >= 140000 rri, #endif slot, estate, #if PG_VERSION_NUM >= 140000 false, /* update */ #endif false, /* noDupErr */ NULL, /* specConflict */ NIL /* arbiterIndexes */ #if PG_VERSION_NUM >= 160000 , false /* onlySummarizing */ #endif ); ExecClearTuple(slot); pfree(tup); /* * If recheck is required, it must have been preformed on the source * relation by now. (All the logical changes we process here are already * committed.) */ list_free(recheck); /* Update the progress information. */ SpinLockAcquire(&MyWorkerTask->mutex); MyWorkerTask->progress.ins++; SpinLockRelease(&MyWorkerTask->mutex); } static void apply_update_or_delete(HeapTuple tup, HeapTuple tup_old, ConcurrentChangeKind change_kind, EState *estate, ScanKey key, int nkeys, Relation ident_index, TupleTableSlot *slot_dst, TupleTableSlot *slot_dst_ind, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, partitions_hash *partitions, TupleConversionMapExt *conv_map) { ResultRelInfo *rri, *rri_old = NULL; /* * Convert the tuple(s) to match the destination table. */ if (conv_map) { tup = convert_tuple_for_dest_table(tup, conv_map); if (tup_old) { Assert(change_kind == CHANGE_UPDATE_NEW); tup_old = convert_tuple_for_dest_table(tup_old, conv_map); } } /* Is the destination table partitioned? */ if (proute) { /* Which partition does the tuple belong to? */ ExecStoreHeapTuple(tup, slot_dst, false); rri = ExecFindPartition(mtstate, mtstate->rootResultRelInfo, proute, slot_dst, estate); ExecClearTuple(slot_dst); if (change_kind == CHANGE_UPDATE_NEW && tup_old) { ExecStoreHeapTuple(tup_old, slot_dst, false); rri_old = ExecFindPartition(mtstate, mtstate->rootResultRelInfo, proute, slot_dst, estate); ExecClearTuple(slot_dst); } } else rri = mtstate->resultRelInfo; /* Is this a cross-partition update? */ if (rri_old && RelationGetRelid(rri_old->ri_RelationDesc) != RelationGetRelid(rri->ri_RelationDesc)) { ItemPointerData ctid; List *recheck; PartitionEntry *entry; /* * Cross-partition update. Delete the old tuple from its partition. */ find_tuple_in_partition(tup_old, rri_old->ri_RelationDesc, partitions, key, nkeys, &ctid); simple_heap_delete(rri_old->ri_RelationDesc, &ctid); /* Update the progress information. */ SpinLockAcquire(&MyWorkerTask->mutex); MyWorkerTask->progress.del++; SpinLockRelease(&MyWorkerTask->mutex); /* * Insert the new tuple into its partition. This might include * conversion to match the partition, see above. */ entry = get_partition_entry(partitions, RelationGetRelid(rri->ri_RelationDesc)); if (entry->conv_map) tup = convert_tuple_for_dest_table(tup, entry->conv_map); ExecStoreHeapTuple(tup, entry->slot, false); table_tuple_insert(rri->ri_RelationDesc, entry->slot, GetCurrentCommandId(true), 0, NULL); #if PG_VERSION_NUM < 140000 estate->es_result_relation_info = rri; #endif /* Update indexes. */ recheck = ExecInsertIndexTuples( #if PG_VERSION_NUM >= 140000 rri, #endif entry->slot, estate, #if PG_VERSION_NUM >= 140000 false, /* update */ #endif false, /* noDupErr */ NULL, /* specConflict */ NIL /* arbiterIndexes */ #if PG_VERSION_NUM >= 160000 , false /* onlySummarizing */ #endif ); ExecClearTuple(entry->slot); /* Update the progress information. */ SpinLockAcquire(&MyWorkerTask->mutex); MyWorkerTask->progress.ins++; SpinLockRelease(&MyWorkerTask->mutex); list_free(recheck); } else { HeapTuple tup_key; ItemPointerData ctid; /* * Both old and new tuple are in the same partition, or the target * table is not partitioned. Find the tuple to be updated or deleted. */ if (change_kind == CHANGE_UPDATE_NEW) tup_key = tup_old != NULL ? tup_old : tup; else { Assert(change_kind == CHANGE_DELETE); Assert(tup_old == NULL); tup_key = tup; } if (partitions) find_tuple_in_partition(tup_key, rri->ri_RelationDesc, partitions, key, nkeys, &ctid); else find_tuple(tup_key, rri->ri_RelationDesc, ident_index, key, nkeys, &ctid, slot_dst_ind); if (change_kind == CHANGE_UPDATE_NEW) { PartitionEntry *entry = NULL; #if PG_VERSION_NUM >= 160000 TU_UpdateIndexes update_indexes; #endif if (partitions) { /* * Make sure the tuple matches the partition. */ entry = get_partition_entry(partitions, RelationGetRelid(rri->ri_RelationDesc)); if (entry->conv_map) tup = convert_tuple_for_dest_table(tup, entry->conv_map); } simple_heap_update(rri->ri_RelationDesc, &ctid, tup #if PG_VERSION_NUM >= 160000 , &update_indexes #endif ); if (!HeapTupleIsHeapOnly(tup)) { TupleTableSlot *slot; List *recheck; slot = entry ? entry->slot : slot_dst; ExecStoreHeapTuple(tup, slot, false); /* * XXX Consider passing update=true, however it requires * es_range_table to be initialized. Is it worth the * complexity? */ recheck = ExecInsertIndexTuples( #if PG_VERSION_NUM >= 140000 rri, #endif slot, estate, #if PG_VERSION_NUM >= 140000 false, /* update */ #endif false, /* noDupErr */ NULL, /* specConflict */ NIL /* arbiterIndexes */ #if PG_VERSION_NUM >= 160000 /* onlySummarizing */ , update_indexes == TU_Summarizing #endif ); ExecClearTuple(slot); list_free(recheck); } /* Update the progress information. */ SpinLockAcquire(&MyWorkerTask->mutex); MyWorkerTask->progress.upd++; SpinLockRelease(&MyWorkerTask->mutex); } else { Assert(change_kind == CHANGE_DELETE); simple_heap_delete(rri->ri_RelationDesc, &ctid); /* Update the progress information. */ SpinLockAcquire(&MyWorkerTask->mutex); MyWorkerTask->progress.del++; SpinLockRelease(&MyWorkerTask->mutex); } } pfree(tup); if (tup_old) pfree(tup_old); } /* * Find tuple whose identity key is passed as 'tup' in relation 'rel' and put * its location into 'ctid'. */ static void find_tuple_in_partition(HeapTuple tup, Relation partition, partitions_hash *partitions, ScanKey key, int nkeys, ItemPointer ctid) { Oid part_oid = RelationGetRelid(partition); HeapTuple tup_mapped = NULL; PartitionEntry *entry; entry = partitions_lookup(partitions, part_oid); if (entry == NULL) elog(ERROR, "identity index not found for partition %u", part_oid); Assert(entry->part_oid == part_oid); /* * Make sure the tuple matches the partition. */ if (entry->conv_map) { /* * convert_tuple_for_dest_table() is not suitable here because we need * to keep the original tuple. XXX Should we add a boolean argument to * the function that indicates whether it should free the original * tuple? */ tup_mapped = pg_rewrite_execute_attr_map_tuple(tup, entry->conv_map); tup = tup_mapped; } find_tuple(tup, partition, entry->ident_index, key, nkeys, ctid, entry->slot_ind); if (tup_mapped) pfree(tup_mapped); } /* * Find tuple whose identity key is passed as 'tup' in relation 'rel' and put * its location into 'ctid'. */ static void find_tuple(HeapTuple tup, Relation rel, Relation ident_index, ScanKey key, int nkeys, ItemPointer ctid, TupleTableSlot *slot_dst_ind) { Form_pg_index ident_form; int2vector *ident_indkey; IndexScanDesc scan; int i; HeapTuple tup_exist; ident_form = ident_index->rd_index; ident_indkey = &ident_form->indkey; scan = index_beginscan(rel, ident_index, GetActiveSnapshot(), #if PG_VERSION_NUM >= 180000 NULL, /* instrument */ #endif nkeys, 0); index_rescan(scan, key, nkeys, NULL, 0); /* Use the incoming tuple to finalize the scan key. */ for (i = 0; i < scan->numberOfKeys; i++) { ScanKey entry; bool isnull; int16 attno_heap; entry = &scan->keyData[i]; attno_heap = ident_indkey->values[i]; entry->sk_argument = heap_getattr(tup, attno_heap, rel->rd_att, &isnull); Assert(!isnull); } if (index_getnext_slot(scan, ForwardScanDirection, slot_dst_ind)) { bool shouldFreeInd; tup_exist = ExecFetchSlotHeapTuple(slot_dst_ind, false, &shouldFreeInd); /* TTSOpsBufferHeapTuple has .get_heap_tuple != NULL. */ Assert(!shouldFreeInd); } else tup_exist = NULL; if (tup_exist == NULL) elog(ERROR, "Failed to find target tuple"); ItemPointerCopy(&tup_exist->t_self, ctid); index_endscan(scan); } static bool processing_time_elapsed(struct timeval *utmost) { struct timeval now; if (utmost == NULL) return false; gettimeofday(&now, NULL); if (now.tv_sec < utmost->tv_sec) return false; if (now.tv_sec > utmost->tv_sec) return true; return now.tv_usec >= utmost->tv_usec; } /* * Convert tuple according to the map and free the original one. */ HeapTuple convert_tuple_for_dest_table(HeapTuple tuple, TupleConversionMapExt *conv_map) { HeapTuple orig = tuple; tuple = pg_rewrite_execute_attr_map_tuple(tuple, conv_map); pfree(orig); return tuple; } void _PG_output_plugin_init(OutputPluginCallbacks *cb) { AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit); cb->startup_cb = plugin_startup; cb->begin_cb = plugin_begin_txn; cb->change_cb = plugin_change; cb->commit_cb = plugin_commit_txn; cb->filter_by_origin_cb = plugin_filter; cb->shutdown_cb = plugin_shutdown; } /* initialize this plugin */ static void plugin_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, bool is_init) { ctx->output_plugin_private = NULL; /* Probably unnecessary, as we don't use the SQL interface ... */ opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT; if (ctx->output_plugin_options != NIL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("This plugin does not expect any options"))); } } static void plugin_shutdown(LogicalDecodingContext *ctx) { } /* * As we don't release the slot during processing of particular table, there's * no room for SQL interface, even for debugging purposes. Therefore we need * neither OutputPluginPrepareWrite() nor OutputPluginWrite() in the plugin * callbacks. (Although we might want to write custom callbacks, this API * seems to be unnecessarily generic for our purposes.) */ /* BEGIN callback */ static void plugin_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn) { } /* COMMIT callback */ static void plugin_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, XLogRecPtr commit_lsn) { } /* * Callback for individual changed tuples */ static void plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change) { DecodingOutputState *dstate; dstate = (DecodingOutputState *) ctx->output_writer_private; /* Only interested in one particular relation. */ if (relation->rd_id != dstate->relid) return; /* Decode entry depending on its type */ switch (change->action) { case REORDER_BUFFER_CHANGE_INSERT: { HeapTuple newtuple; newtuple = change->data.tp.newtuple != NULL ? #if PG_VERSION_NUM >= 170000 change->data.tp.newtuple : NULL; #else &change->data.tp.newtuple->tuple : NULL; #endif /* * Identity checks in the main function should have made this * impossible. */ if (newtuple == NULL) elog(ERROR, "Incomplete insert info."); store_change(ctx, CHANGE_INSERT, newtuple); } break; case REORDER_BUFFER_CHANGE_UPDATE: { HeapTuple oldtuple, newtuple; oldtuple = change->data.tp.oldtuple != NULL ? #if PG_VERSION_NUM >= 170000 change->data.tp.oldtuple : NULL; #else &change->data.tp.oldtuple->tuple : NULL; #endif newtuple = change->data.tp.newtuple != NULL ? #if PG_VERSION_NUM >= 170000 change->data.tp.newtuple : NULL; #else &change->data.tp.newtuple->tuple : NULL; #endif if (newtuple == NULL) elog(ERROR, "Incomplete update info."); if (oldtuple != NULL) store_change(ctx, CHANGE_UPDATE_OLD, oldtuple); store_change(ctx, CHANGE_UPDATE_NEW, newtuple); } break; case REORDER_BUFFER_CHANGE_DELETE: { HeapTuple oldtuple; oldtuple = change->data.tp.oldtuple ? #if PG_VERSION_NUM >= 170000 change->data.tp.oldtuple : NULL; #else &change->data.tp.oldtuple->tuple : NULL; #endif if (oldtuple == NULL) elog(ERROR, "Incomplete delete info."); store_change(ctx, CHANGE_DELETE, oldtuple); } break; default: /* Should not come here */ Assert(0); break; } } /* Store concurrent data change. */ static void store_change(LogicalDecodingContext *ctx, ConcurrentChangeKind kind, HeapTuple tuple) { DecodingOutputState *dstate; char *change_raw; ConcurrentChange *change; MemoryContext oldcontext; bool flattened = false; Size size; Datum values[1]; bool isnull[1]; char *dst; dstate = (DecodingOutputState *) ctx->output_writer_private; /* * ReorderBufferCommit() stores the TOAST chunks in its private memory * context and frees them after having called apply_change(). Therefore we * need flat copy (including TOAST) that we eventually copy into the * memory context which is available to * pg_rewrite_decode_concurrent_changes(). */ if (HeapTupleHasExternal(tuple)) { /* * toast_flatten_tuple_to_datum() might be more convenient but we * don't want the decompression it does. */ tuple = toast_flatten_tuple(tuple, dstate->tupdesc_src); flattened = true; } size = MAXALIGN(VARHDRSZ) + sizeof(ConcurrentChange) + tuple->t_len; /* XXX Isn't there any function / macro to do this? */ if (size >= 0x3FFFFFFF) elog(ERROR, "Change is too big."); oldcontext = MemoryContextSwitchTo(ctx->context); change_raw = (char *) palloc(size); MemoryContextSwitchTo(oldcontext); SET_VARSIZE(change_raw, size); change = (ConcurrentChange *) VARDATA(change_raw); /* * Copy the tuple. * * CAUTION: change->tup_data.t_data must be fixed on retrieval! */ memcpy(&change->tup_data, tuple, sizeof(HeapTupleData)); dst = (char *) change + sizeof(ConcurrentChange); memcpy(dst, tuple->t_data, tuple->t_len); /* The other field. */ change->kind = kind; /* The data has been copied. */ if (flattened) pfree(tuple); /* Store as tuple of 1 bytea column. */ values[0] = PointerGetDatum(change_raw); isnull[0] = false; tuplestore_putvalues(dstate->tstore, dstate->tupdesc_change, values, isnull); /* Accounting. */ dstate->nchanges++; /* Cleanup. */ pfree(change_raw); } /* * Retrieve tuple from a change structure. As for the change, no alignment is * assumed. */ static HeapTuple get_changed_tuple(ConcurrentChange *change) { HeapTupleData tup_data; HeapTuple result; char *src; /* * Ensure alignment before accessing the fields. (This is why we can't use * heap_copytuple() instead of this function.) */ memcpy(&tup_data, &change->tup_data, sizeof(HeapTupleData)); result = (HeapTuple) palloc(HEAPTUPLESIZE + tup_data.t_len); memcpy(result, &tup_data, sizeof(HeapTupleData)); result->t_data = (HeapTupleHeader) ((char *) result + HEAPTUPLESIZE); src = (char *) change + sizeof(ConcurrentChange); memcpy(result->t_data, src, result->t_len); return result; } /* * A filter that recognizes changes produced by the initial load. */ static bool plugin_filter(LogicalDecodingContext *ctx, RepOriginId origin_id) { DecodingOutputState *dstate; dstate = (DecodingOutputState *) ctx->output_writer_private; /* dstate is not initialized during decoding setup - should it be? */ if (dstate && dstate->rorigin != InvalidRepOriginId && origin_id == dstate->rorigin) return true; return false; } pg_rewrite-REL2_0_0/expected/000077500000000000000000000000001504513220300161615ustar00rootroot00000000000000pg_rewrite-REL2_0_0/expected/generated.out000066400000000000000000000036161504513220300206560ustar00rootroot00000000000000-- Generated columns - some meaningful combinations of source and destination -- columns. CREATE TABLE tab7( i int primary key, j int, k int generated always as (i + 1) virtual, l int generated always AS (i + 1) stored, m int generated always AS (i + 1) virtual); CREATE TABLE tab7_new( i int primary key, -- Override the value copied from the source table. j int generated always AS (i - 1) stored, -- Check that the expression is evaluated correctly on the source -- table. k int, -- The same for stored expression. l int, -- Override the value computed on the source table. m int generated always as (i - 1) virtual); INSERT INTO tab7(i, j) VALUES (1, 1); SELECT rewrite_table('tab7', 'tab7_new', 'tab7_orig'); rewrite_table --------------- (1 row) SELECT * FROM tab7; i | j | k | l | m ---+---+---+---+--- 1 | 0 | 2 | 2 | 0 (1 row) CREATE EXTENSION pageinspect; -- HEAP_HASNULL indicates that the value of 'm' hasn't been copied from the -- source table. SELECT raw_flags FROM heap_page_items(get_raw_page('tab7', 0)), LATERAL heap_tuple_infomask_flags(t_infomask, t_infomask2); raw_flags ------------------------------------------------------ {HEAP_HASNULL,HEAP_XMIN_COMMITTED,HEAP_XMAX_INVALID} (1 row) -- For PG < 18, test without VIRTUAL columns. CREATE TABLE tab8( i int primary key, j int, k int generated always AS (i + 1) stored); CREATE TABLE tab8_new( i int primary key, -- Override the value copied from the source table. j int generated always AS (i - 1) stored, -- Check that the expression is evaluated correctly on the source -- table. k int); INSERT INTO tab8(i, j) VALUES (1, 1); SELECT rewrite_table('tab8', 'tab8_new', 'tab8_orig'); rewrite_table --------------- (1 row) SELECT * FROM tab8; i | j | k ---+---+--- 1 | 0 | 2 (1 row) pg_rewrite-REL2_0_0/expected/generated_1.out000066400000000000000000000042371504513220300210760ustar00rootroot00000000000000-- Generated columns - some meaningful combinations of source and destination -- columns. CREATE TABLE tab7( i int primary key, j int, k int generated always as (i + 1) virtual, l int generated always AS (i + 1) stored, m int generated always AS (i + 1) virtual); ERROR: syntax error at or near "virtual" LINE 4: k int generated always as (i + 1) virtual, ^ CREATE TABLE tab7_new( i int primary key, -- Override the value copied from the source table. j int generated always AS (i - 1) stored, -- Check that the expression is evaluated correctly on the source -- table. k int, -- The same for stored expression. l int, -- Override the value computed on the source table. m int generated always as (i - 1) virtual); ERROR: syntax error at or near "virtual" LINE 11: m int generated always as (i - 1) virtual); ^ INSERT INTO tab7(i, j) VALUES (1, 1); ERROR: relation "tab7" does not exist LINE 1: INSERT INTO tab7(i, j) VALUES (1, 1); ^ SELECT rewrite_table('tab7', 'tab7_new', 'tab7_orig'); ERROR: relation "tab7" does not exist SELECT * FROM tab7; ERROR: relation "tab7" does not exist LINE 1: SELECT * FROM tab7; ^ CREATE EXTENSION pageinspect; -- HEAP_HASNULL indicates that the value of 'm' hasn't been copied from the -- source table. SELECT raw_flags FROM heap_page_items(get_raw_page('tab7', 0)), LATERAL heap_tuple_infomask_flags(t_infomask, t_infomask2); ERROR: relation "tab7" does not exist -- For PG < 18, test without VIRTUAL columns. CREATE TABLE tab8( i int primary key, j int, k int generated always AS (i + 1) stored); CREATE TABLE tab8_new( i int primary key, -- Override the value copied from the source table. j int generated always AS (i - 1) stored, -- Check that the expression is evaluated correctly on the source -- table. k int); INSERT INTO tab8(i, j) VALUES (1, 1); SELECT rewrite_table('tab8', 'tab8_new', 'tab8_orig'); rewrite_table --------------- (1 row) SELECT * FROM tab8; i | j | k ---+---+--- 1 | 0 | 2 (1 row) pg_rewrite-REL2_0_0/expected/pg_rewrite.out000066400000000000000000000252201504513220300210620ustar00rootroot00000000000000DROP EXTENSION IF EXISTS pg_rewrite; NOTICE: extension "pg_rewrite" does not exist, skipping CREATE EXTENSION pg_rewrite; CREATE TABLE tab1(i int PRIMARY KEY, j int, k int); -- If a dropped column is encountered, the source tuple should be converted -- so it matches the destination table. ALTER TABLE tab1 DROP COLUMN k; ALTER TABLE tab1 ADD COLUMN k int; INSERT INTO tab1(i, j, k) SELECT i, i / 2, i FROM generate_series(0, 1023) g(i); CREATE TABLE tab1_new(i int PRIMARY KEY, j int, k int) PARTITION BY RANGE(i); CREATE TABLE tab1_new_part_1 PARTITION OF tab1_new FOR VALUES FROM (0) TO (256); CREATE TABLE tab1_new_part_2 PARTITION OF tab1_new FOR VALUES FROM (256) TO (512); CREATE TABLE tab1_new_part_3 PARTITION OF tab1_new FOR VALUES FROM (512) TO (768); CREATE TABLE tab1_new_part_4 PARTITION OF tab1_new FOR VALUES FROM (768) TO (1024); -- Also test handling of constraints that require "manual" validation. ALTER TABLE tab1 ADD CHECK (k >= 0); CREATE TABLE tab1_fk(i int REFERENCES tab1); INSERT INTO tab1_fk(i) VALUES (1); \d tab1 Table "public.tab1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | j | integer | | | k | integer | | | Indexes: "tab1_pkey" PRIMARY KEY, btree (i) Check constraints: "tab1_k_check" CHECK (k >= 0) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1(i) -- Process the table. SELECT rewrite_table('tab1', 'tab1_new', 'tab1_orig'); rewrite_table --------------- (1 row) -- tab1 should now be partitioned. \d tab1 Partitioned table "public.tab1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | j | integer | | | k | integer | | | Partition key: RANGE (i) Indexes: "tab1_new_pkey" PRIMARY KEY, btree (i) Check constraints: "tab1_k_check2" CHECK (k >= 0) NOT VALID Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1(i) NOT VALID Number of partitions: 4 (Use \d+ to list them.) -- Validate the constraints. ALTER TABLE tab1 VALIDATE CONSTRAINT tab1_k_check2; ALTER TABLE tab1_fk VALIDATE CONSTRAINT tab1_fk_i_fkey2; \d tab1 Partitioned table "public.tab1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | j | integer | | | k | integer | | | Partition key: RANGE (i) Indexes: "tab1_new_pkey" PRIMARY KEY, btree (i) Check constraints: "tab1_k_check2" CHECK (k >= 0) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1(i) Number of partitions: 4 (Use \d+ to list them.) EXPLAIN (COSTS off) SELECT * FROM tab1; QUERY PLAN ------------------------------------------ Append -> Seq Scan on tab1_new_part_1 tab1_1 -> Seq Scan on tab1_new_part_2 tab1_2 -> Seq Scan on tab1_new_part_3 tab1_3 -> Seq Scan on tab1_new_part_4 tab1_4 (5 rows) -- Check that the contents has not changed. SELECT count(*) FROM tab1; count ------- 1024 (1 row) SELECT * FROM tab1 t FULL JOIN tab1_orig o ON t.i = o.i WHERE t.i ISNULL OR o.i ISNULL; i | j | k | i | j | k ---+---+---+---+---+--- (0 rows) -- List partitioning CREATE TABLE tab2(i int, j int, PRIMARY KEY (i, j)); INSERT INTO tab2(i, j) SELECT i, j FROM generate_series(1, 4) g(i), generate_series(1, 4) h(j); CREATE TABLE tab2_new(i int, j int, PRIMARY KEY (i, j)) PARTITION BY LIST(i); CREATE TABLE tab2_new_part_1 PARTITION OF tab2_new FOR VALUES IN (1); CREATE TABLE tab2_new_part_2 PARTITION OF tab2_new FOR VALUES IN (2); CREATE TABLE tab2_new_part_3 PARTITION OF tab2_new FOR VALUES IN (3); CREATE TABLE tab2_new_part_4 PARTITION OF tab2_new FOR VALUES IN (4); SELECT rewrite_table('tab2', 'tab2_new', 'tab2_orig'); rewrite_table --------------- (1 row) TABLE tab2_new_part_1; i | j ---+--- 1 | 1 1 | 2 1 | 3 1 | 4 (4 rows) TABLE tab2_new_part_2; i | j ---+--- 2 | 1 2 | 2 2 | 3 2 | 4 (4 rows) TABLE tab2_new_part_3; i | j ---+--- 3 | 1 3 | 2 3 | 3 3 | 4 (4 rows) TABLE tab2_new_part_4; i | j ---+--- 4 | 1 4 | 2 4 | 3 4 | 4 (4 rows) -- Hash partitioning CREATE TABLE tab3(i int, j int, PRIMARY KEY (i, j)); INSERT INTO tab3(i, j) SELECT i, j FROM generate_series(1, 4) g(i), generate_series(1, 4) h(j); CREATE TABLE tab3_new(i int, j int, PRIMARY KEY (i, j)) PARTITION BY HASH(i); CREATE TABLE tab3_new_part_1 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 0); CREATE TABLE tab3_new_part_2 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 1); CREATE TABLE tab3_new_part_3 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 2); CREATE TABLE tab3_new_part_4 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 3); SELECT rewrite_table('tab3', 'tab3_new', 'tab3_orig'); rewrite_table --------------- (1 row) TABLE tab3_new_part_1; i | j ---+--- 1 | 1 1 | 2 1 | 3 1 | 4 (4 rows) TABLE tab3_new_part_2; i | j ---+--- 3 | 1 3 | 2 3 | 3 3 | 4 (4 rows) TABLE tab3_new_part_3; i | j ---+--- 2 | 1 2 | 2 2 | 3 2 | 4 (4 rows) TABLE tab3_new_part_4; i | j ---+--- 4 | 1 4 | 2 4 | 3 4 | 4 (4 rows) -- Change of precision and scale of a numeric data type. CREATE TABLE tab4(i int PRIMARY KEY, j numeric(3, 1)); INSERT INTO tab4(i, j) VALUES (1, 0.1); CREATE TABLE tab4_new(i int PRIMARY KEY, j numeric(4, 2)); TABLE tab4; i | j ---+----- 1 | 0.1 (1 row) SELECT rewrite_table('tab4', 'tab4_new', 'tab4_orig'); rewrite_table --------------- (1 row) TABLE tab4; i | j ---+------ 1 | 0.10 (1 row) -- One more test for "manual" validation of FKs, this time we rewrite the PK -- table. The NOT VALID constraint cannot be used if the FK table is -- partitioned and if PG version is < 18, so we need a separate test. CREATE TABLE tab1_pk(i int primary key); INSERT INTO tab1_pk(i) VALUES (1); CREATE TABLE tab1_pk_new(i bigint primary key); DROP TABLE tab1_fk; CREATE TABLE tab1_fk(i int REFERENCES tab1_pk); INSERT INTO tab1_fk(i) VALUES (1); \d tab1_pk Table "public.tab1_pk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | Indexes: "tab1_pk_pkey" PRIMARY KEY, btree (i) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk(i) SELECT rewrite_table('tab1_pk', 'tab1_pk_new', 'tab1_pk_orig'); rewrite_table --------------- (1 row) \d tab1_pk Table "public.tab1_pk" Column | Type | Collation | Nullable | Default --------+--------+-----------+----------+--------- i | bigint | | not null | Indexes: "tab1_pk_new_pkey" PRIMARY KEY, btree (i) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1_pk(i) NOT VALID ALTER TABLE tab1_fk VALIDATE CONSTRAINT tab1_fk_i_fkey2; \d tab1_pk Table "public.tab1_pk" Column | Type | Collation | Nullable | Default --------+--------+-----------+----------+--------- i | bigint | | not null | Indexes: "tab1_pk_new_pkey" PRIMARY KEY, btree (i) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1_pk(i) -- For the partitioned FK table, test at least that the FK creation is skipped -- (i.e. ERROR saying that NOT VALID is not supported is no raised) DROP TABLE tab1_fk; CREATE TABLE tab1_fk(i int REFERENCES tab1_pk) PARTITION BY RANGE (i); CREATE TABLE tab1_fk_1 PARTITION OF tab1_fk DEFAULT; INSERT INTO tab1_fk(i) VALUES (1); ALTER TABLE tab1_pk_orig RENAME TO tab1_pk_new; TRUNCATE TABLE tab1_pk_new; \d tab1_fk Partitioned table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | | Partition key: RANGE (i) Foreign-key constraints: "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk(i) Number of partitions: 1 (Use \d+ to list them.) SELECT rewrite_table('tab1_pk', 'tab1_pk_new', 'tab1_pk_orig'); rewrite_table --------------- (1 row) -- Note that tab1_fk still references tab1_pk_orig - that's expected. \d tab1_fk Partitioned table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | | Partition key: RANGE (i) Foreign-key constraints: "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk_orig(i) Number of partitions: 1 (Use \d+ to list them.) -- The same once again, but now rewrite the FK table. DROP TABLE tab1_fk; DROP TABLE tab1_pk; ALTER TABLE tab1_pk_orig RENAME TO tab1_pk; CREATE TABLE tab1_fk(i int PRIMARY KEY REFERENCES tab1_pk); INSERT INTO tab1_fk(i) VALUES (1); CREATE TABLE tab1_fk_new(i int PRIMARY KEY) PARTITION BY RANGE (i); CREATE TABLE tab1_fk_new_1 PARTITION OF tab1_fk_new DEFAULT; \d tab1_fk Table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | Indexes: "tab1_fk_pkey" PRIMARY KEY, btree (i) Foreign-key constraints: "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk(i) SELECT rewrite_table('tab1_fk', 'tab1_fk_new', 'tab1_fk_orig'); rewrite_table --------------- (1 row) \d tab1_fk Partitioned table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | Partition key: RANGE (i) Indexes: "tab1_fk_new_pkey" PRIMARY KEY, btree (i) Foreign-key constraints: "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1_pk(i) NOT VALID Number of partitions: 1 (Use \d+ to list them.) -- Check if sequence on the target table is synchronized with that of the -- source table. CREATE TABLE tab5(i int primary key generated always as identity); CREATE TABLE tab5_new(i int primary key generated always as identity); INSERT INTO tab5(i) VALUES (DEFAULT); SELECT rewrite_table('tab5', 'tab5_new', 'tab5_orig'); rewrite_table --------------- (1 row) INSERT INTO tab5(i) VALUES (DEFAULT); SELECT i FROM tab5 ORDER BY i; i --- 1 2 (2 rows) -- The same with serial column. CREATE TABLE tab6(i serial primary key); CREATE TABLE tab6_new(i serial primary key); INSERT INTO tab6(i) VALUES (DEFAULT); SELECT rewrite_table('tab6', 'tab6_new', 'tab6_orig'); rewrite_table --------------- (1 row) INSERT INTO tab6(i) VALUES (DEFAULT); SELECT i FROM tab6 ORDER BY i; i --- 1 2 (2 rows) pg_rewrite-REL2_0_0/expected/pg_rewrite_1.out000066400000000000000000000252011504513220300213010ustar00rootroot00000000000000DROP EXTENSION IF EXISTS pg_rewrite; NOTICE: extension "pg_rewrite" does not exist, skipping CREATE EXTENSION pg_rewrite; CREATE TABLE tab1(i int PRIMARY KEY, j int, k int); -- If a dropped column is encountered, the source tuple should be converted -- so it matches the destination table. ALTER TABLE tab1 DROP COLUMN k; ALTER TABLE tab1 ADD COLUMN k int; INSERT INTO tab1(i, j, k) SELECT i, i / 2, i FROM generate_series(0, 1023) g(i); CREATE TABLE tab1_new(i int PRIMARY KEY, j int, k int) PARTITION BY RANGE(i); CREATE TABLE tab1_new_part_1 PARTITION OF tab1_new FOR VALUES FROM (0) TO (256); CREATE TABLE tab1_new_part_2 PARTITION OF tab1_new FOR VALUES FROM (256) TO (512); CREATE TABLE tab1_new_part_3 PARTITION OF tab1_new FOR VALUES FROM (512) TO (768); CREATE TABLE tab1_new_part_4 PARTITION OF tab1_new FOR VALUES FROM (768) TO (1024); -- Also test handling of constraints that require "manual" validation. ALTER TABLE tab1 ADD CHECK (k >= 0); CREATE TABLE tab1_fk(i int REFERENCES tab1); INSERT INTO tab1_fk(i) VALUES (1); \d tab1 Table "public.tab1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | j | integer | | | k | integer | | | Indexes: "tab1_pkey" PRIMARY KEY, btree (i) Check constraints: "tab1_k_check" CHECK (k >= 0) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1(i) -- Process the table. SELECT rewrite_table('tab1', 'tab1_new', 'tab1_orig'); rewrite_table --------------- (1 row) -- tab1 should now be partitioned. \d tab1 Partitioned table "public.tab1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | j | integer | | | k | integer | | | Partition key: RANGE (i) Indexes: "tab1_new_pkey" PRIMARY KEY, btree (i) Check constraints: "tab1_k_check2" CHECK (k >= 0) NOT VALID Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1(i) NOT VALID Number of partitions: 4 (Use \d+ to list them.) -- Validate the constraints. ALTER TABLE tab1 VALIDATE CONSTRAINT tab1_k_check2; ALTER TABLE tab1_fk VALIDATE CONSTRAINT tab1_fk_i_fkey2; \d tab1 Partitioned table "public.tab1" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | j | integer | | | k | integer | | | Partition key: RANGE (i) Indexes: "tab1_new_pkey" PRIMARY KEY, btree (i) Check constraints: "tab1_k_check2" CHECK (k >= 0) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1(i) Number of partitions: 4 (Use \d+ to list them.) EXPLAIN (COSTS off) SELECT * FROM tab1; QUERY PLAN ------------------------------------------ Append -> Seq Scan on tab1_new_part_1 tab1_1 -> Seq Scan on tab1_new_part_2 tab1_2 -> Seq Scan on tab1_new_part_3 tab1_3 -> Seq Scan on tab1_new_part_4 tab1_4 (5 rows) -- Check that the contents has not changed. SELECT count(*) FROM tab1; count ------- 1024 (1 row) SELECT * FROM tab1 t FULL JOIN tab1_orig o ON t.i = o.i WHERE t.i ISNULL OR o.i ISNULL; i | j | k | i | j | k ---+---+---+---+---+--- (0 rows) -- List partitioning CREATE TABLE tab2(i int, j int, PRIMARY KEY (i, j)); INSERT INTO tab2(i, j) SELECT i, j FROM generate_series(1, 4) g(i), generate_series(1, 4) h(j); CREATE TABLE tab2_new(i int, j int, PRIMARY KEY (i, j)) PARTITION BY LIST(i); CREATE TABLE tab2_new_part_1 PARTITION OF tab2_new FOR VALUES IN (1); CREATE TABLE tab2_new_part_2 PARTITION OF tab2_new FOR VALUES IN (2); CREATE TABLE tab2_new_part_3 PARTITION OF tab2_new FOR VALUES IN (3); CREATE TABLE tab2_new_part_4 PARTITION OF tab2_new FOR VALUES IN (4); SELECT rewrite_table('tab2', 'tab2_new', 'tab2_orig'); rewrite_table --------------- (1 row) TABLE tab2_new_part_1; i | j ---+--- 1 | 1 1 | 2 1 | 3 1 | 4 (4 rows) TABLE tab2_new_part_2; i | j ---+--- 2 | 1 2 | 2 2 | 3 2 | 4 (4 rows) TABLE tab2_new_part_3; i | j ---+--- 3 | 1 3 | 2 3 | 3 3 | 4 (4 rows) TABLE tab2_new_part_4; i | j ---+--- 4 | 1 4 | 2 4 | 3 4 | 4 (4 rows) -- Hash partitioning CREATE TABLE tab3(i int, j int, PRIMARY KEY (i, j)); INSERT INTO tab3(i, j) SELECT i, j FROM generate_series(1, 4) g(i), generate_series(1, 4) h(j); CREATE TABLE tab3_new(i int, j int, PRIMARY KEY (i, j)) PARTITION BY HASH(i); CREATE TABLE tab3_new_part_1 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 0); CREATE TABLE tab3_new_part_2 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 1); CREATE TABLE tab3_new_part_3 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 2); CREATE TABLE tab3_new_part_4 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 3); SELECT rewrite_table('tab3', 'tab3_new', 'tab3_orig'); rewrite_table --------------- (1 row) TABLE tab3_new_part_1; i | j ---+--- 1 | 1 1 | 2 1 | 3 1 | 4 (4 rows) TABLE tab3_new_part_2; i | j ---+--- 3 | 1 3 | 2 3 | 3 3 | 4 (4 rows) TABLE tab3_new_part_3; i | j ---+--- 2 | 1 2 | 2 2 | 3 2 | 4 (4 rows) TABLE tab3_new_part_4; i | j ---+--- 4 | 1 4 | 2 4 | 3 4 | 4 (4 rows) -- Change of precision and scale of a numeric data type. CREATE TABLE tab4(i int PRIMARY KEY, j numeric(3, 1)); INSERT INTO tab4(i, j) VALUES (1, 0.1); CREATE TABLE tab4_new(i int PRIMARY KEY, j numeric(4, 2)); TABLE tab4; i | j ---+----- 1 | 0.1 (1 row) SELECT rewrite_table('tab4', 'tab4_new', 'tab4_orig'); rewrite_table --------------- (1 row) TABLE tab4; i | j ---+------ 1 | 0.10 (1 row) -- One more test for "manual" validation of FKs, this time we rewrite the PK -- table. The NOT VALID constraint cannot be used if the FK table is -- partitioned and if PG version is < 18, so we need a separate test. CREATE TABLE tab1_pk(i int primary key); INSERT INTO tab1_pk(i) VALUES (1); CREATE TABLE tab1_pk_new(i bigint primary key); DROP TABLE tab1_fk; CREATE TABLE tab1_fk(i int REFERENCES tab1_pk); INSERT INTO tab1_fk(i) VALUES (1); \d tab1_pk Table "public.tab1_pk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | Indexes: "tab1_pk_pkey" PRIMARY KEY, btree (i) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk(i) SELECT rewrite_table('tab1_pk', 'tab1_pk_new', 'tab1_pk_orig'); rewrite_table --------------- (1 row) \d tab1_pk Table "public.tab1_pk" Column | Type | Collation | Nullable | Default --------+--------+-----------+----------+--------- i | bigint | | not null | Indexes: "tab1_pk_new_pkey" PRIMARY KEY, btree (i) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1_pk(i) NOT VALID ALTER TABLE tab1_fk VALIDATE CONSTRAINT tab1_fk_i_fkey2; \d tab1_pk Table "public.tab1_pk" Column | Type | Collation | Nullable | Default --------+--------+-----------+----------+--------- i | bigint | | not null | Indexes: "tab1_pk_new_pkey" PRIMARY KEY, btree (i) Referenced by: TABLE "tab1_fk" CONSTRAINT "tab1_fk_i_fkey2" FOREIGN KEY (i) REFERENCES tab1_pk(i) -- For the partitioned FK table, test at least that the FK creation is skipped -- (i.e. ERROR saying that NOT VALID is not supported is no raised) DROP TABLE tab1_fk; CREATE TABLE tab1_fk(i int REFERENCES tab1_pk) PARTITION BY RANGE (i); CREATE TABLE tab1_fk_1 PARTITION OF tab1_fk DEFAULT; INSERT INTO tab1_fk(i) VALUES (1); ALTER TABLE tab1_pk_orig RENAME TO tab1_pk_new; TRUNCATE TABLE tab1_pk_new; \d tab1_fk Partitioned table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | | Partition key: RANGE (i) Foreign-key constraints: "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk(i) Number of partitions: 1 (Use \d+ to list them.) SELECT rewrite_table('tab1_pk', 'tab1_pk_new', 'tab1_pk_orig'); rewrite_table --------------- (1 row) -- Note that tab1_fk still references tab1_pk_orig - that's expected. \d tab1_fk Partitioned table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | | Partition key: RANGE (i) Foreign-key constraints: "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk_orig(i) Number of partitions: 1 (Use \d+ to list them.) -- The same once again, but now rewrite the FK table. DROP TABLE tab1_fk; DROP TABLE tab1_pk; ALTER TABLE tab1_pk_orig RENAME TO tab1_pk; CREATE TABLE tab1_fk(i int PRIMARY KEY REFERENCES tab1_pk); INSERT INTO tab1_fk(i) VALUES (1); CREATE TABLE tab1_fk_new(i int PRIMARY KEY) PARTITION BY RANGE (i); CREATE TABLE tab1_fk_new_1 PARTITION OF tab1_fk_new DEFAULT; \d tab1_fk Table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | Indexes: "tab1_fk_pkey" PRIMARY KEY, btree (i) Foreign-key constraints: "tab1_fk_i_fkey" FOREIGN KEY (i) REFERENCES tab1_pk(i) SELECT rewrite_table('tab1_fk', 'tab1_fk_new', 'tab1_fk_orig'); NOTICE: FOREIGN KEY with NOT VALID option cannot be added to partitioned table rewrite_table --------------- (1 row) \d tab1_fk Partitioned table "public.tab1_fk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- i | integer | | not null | Partition key: RANGE (i) Indexes: "tab1_fk_new_pkey" PRIMARY KEY, btree (i) Number of partitions: 1 (Use \d+ to list them.) -- Check if sequence on the target table is synchronized with that of the -- source table. CREATE TABLE tab5(i int primary key generated always as identity); CREATE TABLE tab5_new(i int primary key generated always as identity); INSERT INTO tab5(i) VALUES (DEFAULT); SELECT rewrite_table('tab5', 'tab5_new', 'tab5_orig'); rewrite_table --------------- (1 row) INSERT INTO tab5(i) VALUES (DEFAULT); SELECT i FROM tab5 ORDER BY i; i --- 1 2 (2 rows) -- The same with serial column. CREATE TABLE tab6(i serial primary key); CREATE TABLE tab6_new(i serial primary key); INSERT INTO tab6(i) VALUES (DEFAULT); SELECT rewrite_table('tab6', 'tab6_new', 'tab6_orig'); rewrite_table --------------- (1 row) INSERT INTO tab6(i) VALUES (DEFAULT); SELECT i FROM tab6 ORDER BY i; i --- 1 2 (2 rows) pg_rewrite-REL2_0_0/expected/pg_rewrite_concurrent.out000066400000000000000000000044001504513220300233210ustar00rootroot00000000000000Parsed test spec with 2 sessions starting permutation: do_rewrite wait_for_before_lock_ip do_changes wakeup_before_lock_ip wait_for_after_commit_ip do_check wakeup_after_commit_ip injection_points_attach ----------------------- (1 row) step do_rewrite: SELECT rewrite_table_nowait('tbl_src', 'tbl_dst', 'tbl_src_old'); rewrite_table_nowait -------------------- (1 row) step wait_for_before_lock_ip: DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-before-lock'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; step do_changes: INSERT INTO tbl_src VALUES (2, 20), (3, 30), (5, 50); -- Update with no identity change. UPDATE tbl_src SET j=0 WHERE i=1; -- Update with identity change. UPDATE tbl_src SET i=6 WHERE i=4; -- Update a row we inserted, to check that the insertion is visible. UPDATE tbl_src SET j=7 WHERE i=2; -- ... and update it again, to check that the update is visible. UPDATE tbl_src SET j=8 WHERE j=7; -- Delete. DELETE FROM tbl_src WHERE i=7; step wakeup_before_lock_ip: SELECT injection_points_wakeup('pg_rewrite-before-lock'); injection_points_wakeup ----------------------- (1 row) step wait_for_after_commit_ip: DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-after-commit'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; step do_check: TABLE pg_rewrite_progress; SELECT i, j, k, l FROM tbl_src ORDER BY i, j; src_table|dst_table|src_table_new|ins_initial|ins|upd|del ---------+---------+-------------+-----------+---+---+--- tbl_src |tbl_dst |tbl_src_old | 3| 3| 4| 1 (1 row) i| j| k| l -+--+---+--- 1| 0| 0| 0 2| 8| -8| -8 3|30|-30|-30 5|50|-50|-50 6|40|-40|-40 (5 rows) step wakeup_after_commit_ip: SELECT injection_points_wakeup('pg_rewrite-after-commit'); injection_points_wakeup ----------------------- (1 row) injection_points_detach ----------------------- (1 row) pg_rewrite-REL2_0_0/expected/pg_rewrite_concurrent_partition.out000066400000000000000000000043351504513220300254210ustar00rootroot00000000000000Parsed test spec with 2 sessions starting permutation: do_rewrite wait_for_before_lock_ip do_changes wakeup_before_lock_ip wait_for_after_commit_ip do_check wakeup_after_commit_ip injection_points_attach ----------------------- (1 row) step do_rewrite: SELECT rewrite_table_nowait('tbl_src', 'tbl_dst', 'tbl_src_old'); rewrite_table_nowait -------------------- (1 row) step wait_for_before_lock_ip: DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-before-lock'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; step do_changes: -- Insert one row into each partition. INSERT INTO tbl_src VALUES (2, 20), (3, 30), (5, 50); -- Update with no identity change. UPDATE tbl_src SET j=0 WHERE i=1; -- Update with identity change but within the same partition. UPDATE tbl_src SET i=6 WHERE i=5; -- Cross-partition update. UPDATE tbl_src SET i=7 WHERE i=3; -- Update a row we inserted and updated, to check that it's visible. UPDATE tbl_src SET j=4 WHERE i=7; -- Delete. DELETE FROM tbl_src WHERE i=4; step wakeup_before_lock_ip: SELECT injection_points_wakeup('pg_rewrite-before-lock'); injection_points_wakeup ----------------------- (1 row) step wait_for_after_commit_ip: DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-after-commit'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$ step do_check: TABLE pg_rewrite_progress; SELECT i, j FROM tbl_src ORDER BY i, j; src_table|dst_table|src_table_new|ins_initial|ins|upd|del ---------+---------+-------------+-----------+---+---+--- tbl_src |tbl_dst |tbl_src_old | 2| 4| 3| 2 (1 row) i| j -+-- 1| 0 2|20 6|50 7| 4 (4 rows) step wakeup_after_commit_ip: SELECT injection_points_wakeup('pg_rewrite-after-commit'); injection_points_wakeup ----------------------- (1 row) injection_points_detach ----------------------- (1 row) pg_rewrite-REL2_0_0/expected/pg_rewrite_concurrent_toast.out000066400000000000000000000042701504513220300245400ustar00rootroot00000000000000Parsed test spec with 2 sessions starting permutation: do_rewrite wait_for_before_lock_ip do_changes wakeup_before_lock_ip wait_for_after_commit_ip do_check wakeup_after_commit_ip injection_points_attach ----------------------- (1 row) step do_rewrite: SELECT rewrite_table_nowait('tbl_src', 'tbl_dst', 'tbl_src_old'); rewrite_table_nowait -------------------- (1 row) step wait_for_before_lock_ip: DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-before-lock'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; step do_changes: INSERT INTO tbl_src(i, t) SELECT 5, string_agg(random()::text, '') FROM generate_series(1, 200) h(y); UPDATE tbl_src SET t = t || 'x' WHERE i = 1; step wakeup_before_lock_ip: SELECT injection_points_wakeup('pg_rewrite-before-lock'); injection_points_wakeup ----------------------- (1 row) step wait_for_after_commit_ip: DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-after-commit'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; step do_check: TABLE pg_rewrite_progress; -- Each row should contain TOASTed value. SELECT count(*) FROM tbl_src WHERE pg_column_toast_chunk_id(t) ISNULL; -- The contents of the new table should be identical to that of the old -- one. SELECT count(*) FROM tbl_src t1 JOIN tbl_src_old t2 ON t1.i = t2.i WHERE t1.t <> t2.t; src_table|dst_table|src_table_new|ins_initial|ins|upd|del ---------+---------+-------------+-----------+---+---+--- tbl_src |tbl_dst |tbl_src_old | 2| 1| 1| 0 (1 row) count ----- 0 (1 row) count ----- 0 (1 row) step wakeup_after_commit_ip: SELECT injection_points_wakeup('pg_rewrite-after-commit'); injection_points_wakeup ----------------------- (1 row) injection_points_detach ----------------------- (1 row) pg_rewrite-REL2_0_0/pg_rewrite--1.0--1.1.sql000066400000000000000000000020421504513220300201710ustar00rootroot00000000000000/* pg_rewrite--1.0--1.1.sql */ -- complain if script is sourced in psql, rather than via ALTER EXTENSION \echo Use "ALTER EXTENSION pg_rewrite UPDATE TO '1.1'" to load this file. \quit DROP FUNCTION partition_table(text, text, text); CREATE FUNCTION partition_table( src_table text, dst_table text, src_table_new text) RETURNS void AS 'MODULE_PATHNAME', 'partition_table_new' LANGUAGE C STRICT; CREATE FUNCTION pg_rewrite_get_task_list() RETURNS TABLE ( tabschema_src name, tabname_src name, tabschema_dst name, tabname_dst name, tabname_src_new name, ins_initial bigint, ins bigint, upd bigint, del bigint) AS 'MODULE_PATHNAME', 'pg_rewrite_get_task_list' LANGUAGE C; -- The column names should match the arguments of the partition_table() -- function. CREATE VIEW pg_rewrite_progress AS SELECT COALESCE(tabschema_src || '.', '') || tabname_src AS src_table, COALESCE(tabschema_dst || '.', '') || tabname_dst AS dst_table, tabname_src_new AS src_table_new, ins_initial, ins, upd, del FROM pg_rewrite_get_task_list(); pg_rewrite-REL2_0_0/pg_rewrite--1.0.sql000066400000000000000000000005231504513220300176210ustar00rootroot00000000000000/* pg_rewrite--1.0.sql */ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_rewrite" to load this file. \quit CREATE FUNCTION partition_table( src_table text, dst_table text, src_table_new text) RETURNS void AS 'MODULE_PATHNAME', 'partition_table' LANGUAGE C; pg_rewrite-REL2_0_0/pg_rewrite--1.1--1.2.sql000066400000000000000000000011301504513220300201700ustar00rootroot00000000000000/* pg_rewrite--1.1--1.2.sql */ -- complain if script is sourced in psql, rather than via ALTER EXTENSION \echo Use "ALTER EXTENSION pg_rewrite UPDATE TO '1.2'" to load this file. \quit DROP FUNCTION partition_table(text, text, text); CREATE FUNCTION rewrite_table( src_table text, dst_table text, src_table_new text) RETURNS void AS 'MODULE_PATHNAME', 'rewrite_table' LANGUAGE C STRICT; CREATE FUNCTION rewrite_table_nowait( src_table text, dst_table text, src_table_new text) RETURNS void AS 'MODULE_PATHNAME', 'rewrite_table_nowait' LANGUAGE C STRICT; pg_rewrite-REL2_0_0/pg_rewrite--1.2--2.0.sql000066400000000000000000000002721504513220300201760ustar00rootroot00000000000000/* pg_rewrite--1.2--2.0.sql */ -- complain if script is sourced in psql, rather than via ALTER EXTENSION \echo Use "ALTER EXTENSION pg_rewrite UPDATE TO '2.0'" to load this file. \quit pg_rewrite-REL2_0_0/pg_rewrite--1.3--2.0.sql000066400000000000000000000006441504513220300202020ustar00rootroot00000000000000/* pg_rewrite--1.3--2.0.sql */ -- complain if script is sourced in psql, rather than via ALTER EXTENSION \echo Use "ALTER EXTENSION pg_rewrite UPDATE TO '2.0'" to load this file. \quit DROP FUNCTION IF EXISTS rewrite_table_nowait; CREATE FUNCTION rewrite_table_nowait( src_table text, dst_table text, src_table_new text) RETURNS void AS 'MODULE_PATHNAME', 'rewrite_table_nowait' LANGUAGE C STRICT;pg_rewrite-REL2_0_0/pg_rewrite.c000066400000000000000000003202541504513220300167010ustar00rootroot00000000000000/*---------------------------------------------------------------- * * pg_rewrite.c * Tools for maintenance that requires table rewriting. * * Copyright (c) 2021-2025, Cybertec PostgreSQL International GmbH * *---------------------------------------------------------------- */ #include "pg_rewrite.h" #if PG_VERSION_NUM < 130000 #error "PostgreSQL version 13 or higher is required" #endif #include "access/heaptoast.h" #include "access/multixact.h" #include "access/sysattr.h" #include "access/tupdesc_details.h" #if PG_VERSION_NUM >= 150000 #include "access/xloginsert.h" #endif #include "access/xlogutils.h" #include "catalog/catalog.h" #include "catalog/heap.h" #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/objectaddress.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" #include "catalog/pg_constraint.h" #include "catalog/pg_control.h" #include "catalog/pg_depend.h" #include "catalog/pg_extension.h" #include "catalog/pg_type.h" #include "catalog/pg_tablespace.h" #include "catalog/toasting.h" #include "commands/dbcommands.h" #include "commands/extension.h" #include "commands/tablecmds.h" #include "commands/tablespace.h" #include "executor/executor.h" #include "executor/execPartition.h" #include "executor/spi.h" #include "funcapi.h" #include "lib/stringinfo.h" #include "nodes/primnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" #include "parser/parse_coerce.h" #include "parser/parse_collate.h" #include "replication/snapbuild.h" #include "partitioning/partdesc.h" #include "storage/bufmgr.h" #include "storage/freespace.h" #include "storage/ipc.h" #include "storage/lmgr.h" #include "storage/proc.h" #include "storage/smgr.h" #include "storage/standbydefs.h" #include "rewrite/rewriteHandler.h" #include "tcop/tcopprot.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/guc.h" #if PG_VERSION_NUM >= 170000 #include "utils/injection_point.h" #endif #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" #include "utils/ruleutils.h" #include "utils/syscache.h" #include "utils/varlena.h" #ifdef PG_MODULE_MAGIC_EXT PG_MODULE_MAGIC_EXT(.name = "pg_rewrite", .version = "2.0"); #else PG_MODULE_MAGIC; #endif #define REPL_SLOT_BASE_NAME "pg_rewrite_slot_" #define REPL_PLUGIN_NAME "pg_rewrite" /* * Information needed to set sequences belonging the destination table * according to the corresponding sequences of the source table. */ typedef struct SequenceValue { NameData attname; int64 last_value; } SequenceValue; static void rewrite_table_impl(char *relschema_src, char *relname_src, char *relname_new, char *relschema_dst, char *relname_dst); static Relation get_identity_index(Relation rel_dst, Relation rel_src); static partitions_hash *get_partitions(Relation rel_src, Relation rel_dst, int *nparts, Relation **parts_dst_p, ScanKey *ident_key_p, int *ident_key_nentries); static List *get_sequences(Relation rel); static List *getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype); static void set_sequences(Relation rel, List *seqs_src); /* The WAL segment being decoded. */ XLogSegNo rewrite_current_segment = 0; static void worker_shmem_request(void); static void worker_shmem_startup(void); static void worker_shmem_shutdown(int code, Datum arg); static void relation_rewrite_get_args(PG_FUNCTION_ARGS, RangeVar **rv_src_p, RangeVar **rv_src_new_p, RangeVar **rv_dst_p); static WorkerTask *get_task(int *idx, char *relschema, char *relname, bool nowait); static void initialize_worker(BackgroundWorker *worker, int task_idx); static void run_worker(BackgroundWorker *worker, WorkerTask *task, bool nowait); static void send_message(WorkerTask *task, int elevel, const char *message, const char *detail); static void check_prerequisites(Relation rel); static LogicalDecodingContext *setup_decoding(Relation rel); static void decoding_cleanup(LogicalDecodingContext *ctx); static ModifyTableState *get_modify_table_state(EState *estate, Relation rel, CmdType operation); static void free_modify_table_state(ModifyTableState *mtstate); static Snapshot build_historic_snapshot(SnapBuild *builder); static void perform_initial_load(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, Relation rel_src, Snapshot snap_hist, Relation rel_dst, partitions_hash *partitions, LogicalDecodingContext *ctx, TupleConversionMapExt *conv_map); static ScanKey build_identity_key(Relation ident_idx_rel, int *nentries); static bool perform_final_merge(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, Relation rel_src, ScanKey ident_key, int ident_key_nentries, Relation ident_index, TupleTableSlot *slot_dst_ind, LogicalDecodingContext *ctx, partitions_hash *ident_indexes, TupleConversionMapExt *conv_map); static void close_partitions(partitions_hash *partitions); static AttrMapExt *make_attrmap_ext(int maplen); static void free_attrmap_ext(AttrMapExt *map); static TupleConversionMapExt *convert_tuples_by_name_ext(Relation rel_src, Relation rel_dst); static AttrMapExt *build_attrmap_by_name_if_req_ext(Relation rel_src, Relation rel_dst); static AttrMapExt *build_attrmap_by_name_ext(Relation rel_src, Relation rel_dst); static bool check_attrmap_match_ext(TupleDesc indesc, TupleDesc outdesc, AttrMapExt *attrMap); static TupleConversionMapExt *convert_tuples_by_name_attrmap_ext(TupleDesc indesc, TupleDesc outdesc, AttrMapExt *attrMap); static void free_conversion_map_ext(TupleConversionMapExt *map); static void copy_constraints(Oid relid_dst, const char *relname_dst, Oid relid_src); static void dump_fk_constraint(HeapTuple tup, Oid relid_dst, const char *relname_dst, Oid relid_src, StringInfo buf); static void dump_check_constraint(Oid relid_dst, const char *relname_dst, HeapTuple tup, StringInfo buf); #if PG_VERSION_NUM >= 180000 static void dump_null_constraint(Oid relid_dst, const char *relname_dst, HeapTuple tup, StringInfo buf); #endif static void dump_constraint_common(const char *nsp, const char *relname, Form_pg_constraint con, StringInfo buf); static int decompile_column_index_array(Datum column_index_array, Oid relId, StringInfo buf); static Node *build_generation_expression_ext(Relation rel, int attrno); /* * The maximum time to hold AccessExclusiveLock on the source table during the * final processing. Note that it only pg_rewrite_process_concurrent_changes() * execution time is included here. */ static int rewrite_max_xlock_time = 0; #if PG_VERSION_NUM >= 150000 static shmem_request_hook_type prev_shmem_request_hook = NULL; #endif static shmem_startup_hook_type prev_shmem_startup_hook = NULL; void _PG_init(void) { if (!process_shared_preload_libraries_in_progress) ereport(ERROR, (errmsg("pg_rewrite must be loaded via shared_preload_libraries"))); #if PG_VERSION_NUM >= 150000 prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = worker_shmem_request; #else worker_shmem_request(); #endif prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = worker_shmem_startup; DefineCustomIntVariable("rewrite.max_xlock_time", "The maximum time the processed table may be locked exclusively.", "The source table is locked exclusively during the final stage of " "processing. If the lock time should exceed this value, the lock is " "released and the final stage is retried a few more times.", &rewrite_max_xlock_time, 0, 0, INT_MAX, PGC_USERSET, GUC_UNIT_MS, NULL, NULL, NULL); } #define REPLORIGIN_NAME_PATTERN "pg_rewrite_%u" /* * The original implementation would certainly fail on PG 16 and higher, due * to the commit 240e0dbacd (in the master branch) - this commit makes it * impossible to invoke our functionality via the PG executor. It's not worth * supporting lower versions of pg_rewrite on lower versions of PG server. We * keep the symbol in the library so that the upgrade path works. */ extern Datum partition_table(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(partition_table); Datum partition_table(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("the function is no longer supported"), errhint("please run \"ALTER EXTENSION pg_rewrite UPDATE\""))); PG_RETURN_VOID(); } /* * Likewise, keep the symbol because the upgrade path to 1.3 (or higher) * requires that. */ extern Datum partition_table_new(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(partition_table_new); Datum partition_table_new(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("the function is no longer supported"), errhint("please run \"ALTER EXTENSION pg_rewrite UPDATE\""))); PG_RETURN_VOID(); } /* Pointer to task array in the shared memory, available in all backends. */ static WorkerTask *workerTasks = NULL; /* Each worker stores here the pointer to its task in the shared memory. */ WorkerTask *MyWorkerTask = NULL; static void interrupt_worker(WorkerTask *task) { SpinLockAcquire(&task->mutex); task->exit_requested = true; SpinLockRelease(&task->mutex); } static void release_task(WorkerTask *task, bool worker) { if (worker) { SpinLockAcquire(&task->mutex); /* * First, handle special case that can happen in regression tests. If * rewrite_table_nowait() gets cancelled before the worker got its * MyDatabaseId assigned, 'task' slot can leak (note that * rewrite_table_nowait() does not release the task in this case). We * can release the task regardless of MyDatabaseId because * pg_rewrite_concurrent.spec should not launch a new worker (and thus * reuse the task) before the existing one exited. */ if (task->nowait) task->dbid = InvalidOid; /* * Otherwise, worker must not release the task because the backend can * be interested in its contents. */ /* * However, the worker always should clear the fields it set. */ task->pid = InvalidPid; task->exit_requested = false; SpinLockRelease(&task->mutex); return; } /* * The following should only be performed by the backend, after the worker * has exited. */ SpinLockAcquire(&task->mutex); Assert(OidIsValid(task->dbid)); task->dbid = InvalidOid; SpinLockRelease(&task->mutex); } static Size worker_shmem_size(void) { return MAX_TASKS * sizeof(WorkerTask); } static void worker_shmem_request(void) { /* With lower PG versions this function is called from _PG_init(). */ #if PG_VERSION_NUM >= 150000 if (prev_shmem_request_hook) prev_shmem_request_hook(); #endif /* PG_VERSION_NUM >= 150000 */ RequestAddinShmemSpace(worker_shmem_size()); } static void worker_shmem_startup(void) { bool found; if (prev_shmem_startup_hook) prev_shmem_startup_hook(); LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); workerTasks = ShmemInitStruct("pg_rewrite", worker_shmem_size(), &found); if (!found) { int i; for (i = 0; i < MAX_TASKS; i++) { WorkerTask *task = &workerTasks[i]; task->dbid = InvalidOid; task->roleid = InvalidOid; task->pid = InvalidPid; task->exit_requested = false; SpinLockInit(&task->mutex); } } LWLockRelease(AddinShmemInitLock); } static void worker_shmem_shutdown(int code, Datum arg) { if (MyWorkerTask) release_task(MyWorkerTask, true); } static void relation_rewrite_get_args(PG_FUNCTION_ARGS, RangeVar **rv_src_p, RangeVar **rv_src_new_p, RangeVar **rv_dst_p) { text *rel_src_t, *rel_src_new_t, *rel_dst_t; RangeVar *rv_src, *rv_src_new, *rv_dst; rel_src_t = PG_GETARG_TEXT_PP(0); rv_src = makeRangeVarFromNameList(textToQualifiedNameList(rel_src_t)); rel_dst_t = PG_GETARG_TEXT_PP(1); rv_dst = makeRangeVarFromNameList(textToQualifiedNameList(rel_dst_t)); rel_src_new_t = PG_GETARG_TEXT_PP(2); rv_src_new = makeRangeVarFromNameList(textToQualifiedNameList(rel_src_new_t)); if (rv_src->catalogname || rv_dst->catalogname || rv_src_new->catalogname) ereport(ERROR, (errmsg("relation may only be qualified by schema, not by database"))); /* * Technically it's possible to move the source relation to another schema * but don't bother for this version. */ if (rv_src_new->schemaname) ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), (errmsg("the new source relation name may not be qualified")))); *rv_src_p = rv_src; *rv_src_new_p = rv_src_new; *rv_dst_p = rv_dst; } /* * Find a free task structure and initialize the common fields. */ static WorkerTask * get_task(int *idx, char *relschema, char *relname, bool nowait) { int i; WorkerTask *task = NULL; bool found = false; for (i = 0; i < MAX_TASKS; i++) { task = &workerTasks[i]; SpinLockAcquire(&task->mutex); if (task->dbid == InvalidOid && task->pid == InvalidPid) { TaskProgress *progress = &task->progress; /* Make sure that no other backend can use the task. */ task->dbid = MyDatabaseId; progress->ins_initial = 0; progress->ins = 0; progress->upd = 0; progress->del = 0; found = true; } SpinLockRelease(&task->mutex); if (found) break; } if (!found) ereport(ERROR, (errmsg("too many concurrent tasks in progress"))); /* Finalize the task. */ task->roleid = GetUserId(); task->exit_requested = false; if (relschema) namestrcpy(&task->relschema, relschema); else NameStr(task->relschema)[0] = '\0'; namestrcpy(&task->relname, relname); task->msg[0] = '\0'; task->msg_detail[0] = '\0'; task->elevel = -1; task->nowait = nowait; task->max_xlock_time = rewrite_max_xlock_time; *idx = i; return task; } static void initialize_worker(BackgroundWorker *worker, int task_idx) { char *dbname; worker->bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; worker->bgw_start_time = BgWorkerStart_RecoveryFinished; worker->bgw_restart_time = BGW_NEVER_RESTART; sprintf(worker->bgw_library_name, "pg_rewrite"); sprintf(worker->bgw_function_name, "rewrite_worker_main"); /* * XXX The function can throw ERROR but the database should really exist, * so no need to put this code in the PG_TRY block. */ dbname = get_database_name(MyDatabaseId); snprintf(worker->bgw_name, BGW_MAXLEN, "pg_rewrite worker for database %s", dbname); snprintf(worker->bgw_type, BGW_MAXLEN, "pg_rewrite worker"); worker->bgw_main_arg = (Datum) task_idx; worker->bgw_notify_pid = MyProcPid; } static void run_worker(BackgroundWorker *worker, WorkerTask *task, bool nowait) { BackgroundWorkerHandle *handle; BgwHandleStatus status; pid_t pid; char *msg = NULL; char *msg_detail = NULL; int elevel = -1; /* * Start the worker. Avoid leaking the task if the function ends due to * ERROR. */ PG_TRY(); { if (!RegisterDynamicBackgroundWorker(worker, &handle)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("could not register background process"), errhint("More details may be available in the server log."))); status = WaitForBackgroundWorkerStartup(handle, &pid); } PG_CATCH(); { /* * It seems possible that the worker is trying to start even if we end * up here - at least when WaitForBackgroundWorkerStartup() got * interrupted. */ interrupt_worker(task); release_task(task, false); PG_RE_THROW(); } PG_END_TRY(); if (status == BGWH_STOPPED) { /* Work already done? */ goto done; } else if (status == BGWH_POSTMASTER_DIED) { ereport(ERROR, (errmsg("could not start background worker because the postmaster died"), errhint("More details may be available in the server log."))); /* No need to release the task in the shared memory. */ } /* * WaitForBackgroundWorkerStartup() should not return * BGWH_NOT_YET_STARTED. */ Assert(status == BGWH_STARTED); if (nowait) /* The worker should take care of releasing the task. */ return; PG_TRY(); { status = WaitForBackgroundWorkerShutdown(handle); } PG_CATCH(); { /* * Make sure the worker stops. Interrupt received from the user is the * typical use case. */ interrupt_worker(task); release_task(task, false); PG_RE_THROW(); } PG_END_TRY(); if (status == BGWH_POSTMASTER_DIED) { ereport(ERROR, (errmsg("the postmaster died before the background worker could finish"), errhint("More details may be available in the server log."))); /* No need to release the task in the shared memory. */ } /* * WaitForBackgroundWorkerShutdown() should not return anything else. */ Assert(status == BGWH_STOPPED); done: if (strlen(task->msg) > 0) { msg = pstrdup(task->msg); elevel = task->elevel; } if (strlen(task->msg_detail) > 0) msg_detail = pstrdup(task->msg_detail); release_task(task, false); /* Report the worker's ERROR in the backend. */ if (msg) { if (msg_detail) ereport(elevel, (errmsg("%s", msg), errdetail("%s", msg_detail))); else ereport(elevel, (errmsg("%s", msg))); } } /* * Send log message from the worker to the backend that launched it. * * Currently we only copy 'message' and 'detail. More fields can be added to * WorkerTask if needed. Another limitation is that if the worker sends * multiple messages, the backend only receives the last one. * * (Ideally we should use the message queue like parallel workers do, but the * related PG core functions have some parallel worker specific arguments.) */ static void send_message(WorkerTask *task, int elevel, const char *message, const char *detail) { strlcpy(task->msg, message, MAX_ERR_MSG_LEN); task->elevel = elevel; if (detail && strlen(detail) > 0) strlcpy(task->msg_detail, detail, MAX_ERR_MSG_LEN); else /* * Message with elevel < ERROR could already have been written here. */ task->msg_detail[0] = '\0'; } /* PG >= 14 does define this macro. */ #if PG_VERSION_NUM < 140000 #define RelationIsPermanent(relation) \ ((relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT) #endif /* * Start the background worker and wait until it exits. */ extern Datum rewrite_table(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(rewrite_table); Datum rewrite_table(PG_FUNCTION_ARGS) { RangeVar *rv_src, *rv_src_new, *rv_dst; BackgroundWorker worker; WorkerTask *task; int task_idx; relation_rewrite_get_args(fcinfo, &rv_src, &rv_src_new, &rv_dst); task = get_task(&task_idx, rv_src->schemaname, rv_src->relname, false); Assert(task_idx < MAX_TASKS); /* Specify the relation to be processed. */ if (rv_dst->schemaname) namestrcpy(&task->relschema_dst, rv_dst->schemaname); else NameStr(task->relschema_dst)[0] = '\0'; namestrcpy(&task->relname_dst, rv_dst->relname); namestrcpy(&task->relname_new, rv_src_new->relname); initialize_worker(&worker, task_idx); run_worker(&worker, task, false); PG_RETURN_VOID(); } /* * See pg_rewrite_concurrent.spec for information why this function is needed. */ extern Datum rewrite_table_nowait(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(rewrite_table_nowait); Datum rewrite_table_nowait(PG_FUNCTION_ARGS) { RangeVar *rv_src, *rv_src_new, *rv_dst; BackgroundWorker worker; WorkerTask *task; int task_idx; relation_rewrite_get_args(fcinfo, &rv_src, &rv_src_new, &rv_dst); task = get_task(&task_idx, rv_src->schemaname, rv_src->relname, true); Assert(task_idx < MAX_TASKS); /* Specify the relation to be processed. */ if (rv_dst->schemaname) namestrcpy(&task->relschema_dst, rv_dst->schemaname); else NameStr(task->relschema_dst)[0] = '\0'; namestrcpy(&task->relname_dst, rv_dst->relname); namestrcpy(&task->relname_new, rv_src_new->relname); initialize_worker(&worker, task_idx); run_worker(&worker, task, true); PG_RETURN_VOID(); } void rewrite_worker_main(Datum main_arg) { Datum arg; int i; Oid dbid, roleid; char *relschema, *relname, *relname_new, *relschema_dst, *relname_dst; WorkerTask *task; /* The worker should do its cleanup when exiting. */ before_shmem_exit(worker_shmem_shutdown, (Datum) 0); /* * The standard handlers for SIGTERM and SIGQUIT are fine, see * bgworker.c. */ BackgroundWorkerUnblockSignals(); /* Retrieve task index. */ Assert(MyBgworkerEntry != NULL); arg = MyBgworkerEntry->bgw_main_arg; i = DatumGetInt32(arg); Assert(i >= 0 && i < MAX_TASKS); Assert(MyWorkerTask == NULL); task = MyWorkerTask = &workerTasks[i]; /* * The task should be fully initialized before the backend registers the * worker. Let's copy the arguments so that we have a consistent view - * see the explanation below. */ relschema = NameStr(task->relschema); relschema = *relschema != '\0' ? pstrdup(relschema) : NULL; relname = pstrdup(NameStr(task->relname)); relname_new = pstrdup(NameStr(task->relname_new)); relschema_dst = NameStr(task->relschema_dst); relschema_dst = *relschema_dst != '\0' ? pstrdup(relschema_dst) : NULL; relname_dst = pstrdup(NameStr(task->relname_dst)); /* * Get the information provided by the backend and set our pid. */ SpinLockAcquire(&MyWorkerTask->mutex); dbid = MyWorkerTask->dbid; Assert(MyWorkerTask->roleid != InvalidOid); roleid = MyWorkerTask->roleid; task->pid = MyProcPid; SpinLockRelease(&MyWorkerTask->mutex); /* * Has the "owning" backend of this worker exited too early? */ if (!OidIsValid(dbid)) { ereport(DEBUG1, (errmsg("task cancelled before the worker could start"))); return; } /* * If the backend exits later (w/o waiting for the worker's exit), that * backend's ERRORs (which include interrupts) should make the worker stop * (via interrupt_worker()). */ BackgroundWorkerInitializeConnectionByOid(dbid, roleid, 0); /* Do the actual work. */ StartTransactionCommand(); PG_TRY(); { rewrite_table_impl(relschema, relname, relname_new, relschema_dst, relname_dst); CommitTransactionCommand(); /* * In regression tests, use this injection point to check that * the changes are visible by other transactions. */ #if PG_VERSION_NUM >= 180000 INJECTION_POINT("pg_rewrite-after-commit", NULL); #elif PG_VERSION_NUM >= 170000 INJECTION_POINT("pg_rewrite-after-commit"); #endif } PG_CATCH(); { MemoryContext old_context = CurrentMemoryContext; ErrorData *edata; /* * If the backend is not waiting for our exit, make sure the error is * logged. */ if (MyWorkerTask->nowait) PG_RE_THROW(); HOLD_INTERRUPTS(); /* * CopyErrorData() requires the context to be different from * ErrorContext. */ MemoryContextSwitchTo(TopMemoryContext); edata = CopyErrorData(); MemoryContextSwitchTo(old_context); /* * The following shouldn't be necessary because the worker isn't going * to do anything else, but cleanup is just a good practice. * * XXX Should we re-throw the error instead of doing the cleanup? Not * sure, the error message would then appear twice in the log. */ FlushErrorState(); /* Not done by AbortTransaction(). */ if (MyReplicationSlot != NULL) ReplicationSlotRelease(); /* * Likewise, there seems to be no automatic cleanup of the origin, so * do it here. The insertion into the ReplicationOriginRelationId * catalog will be rolled back due to the transaction abort. */ if (replorigin_session_origin != InvalidRepOriginId) replorigin_session_origin = InvalidRepOriginId; AbortOutOfAnyTransaction(); send_message(task, ERROR, edata->message, edata->detail); FreeErrorData(edata); } PG_END_TRY(); } /* * A substitute for CHECK_FOR_INTERRUPRS. * * procsignal_sigusr1_handler does not support signaling from a backend to a * non-parallel worker (see the values of ProcSignalReason), so the worker * cannot use CHECK_FOR_INTERRUPTS. Let's use shared memory to tell the worker * that it should exit. (SIGTERM would terminate the worker easily, but due * to race conditions we could terminate another backend / worker which * already managed to reuse this worker's PID.) */ void pg_rewrite_exit_if_requested(void) { bool exit_requested; SpinLockAcquire(&MyWorkerTask->mutex); exit_requested = MyWorkerTask->exit_requested; SpinLockRelease(&MyWorkerTask->mutex); if (!exit_requested) return; /* * There seems to be no automatic cleanup of the origin, so do it here. * The insertion into the ReplicationOriginRelationId catalog will be * rolled back due to the transaction abort. */ if (replorigin_session_origin != InvalidRepOriginId) replorigin_session_origin = InvalidRepOriginId; /* * Message similar to that in ProcessInterrupts(), but ERROR is * sufficient here. rewrite_worker_main() should catch it. */ ereport(ERROR, (errcode(ERRCODE_ADMIN_SHUTDOWN), errmsg("terminating pg_rewrite background worker due to administrator command"))); } /* * Perform the rewriting. * * The function is executed by a background worker. We do not catch ERRORs * here, they will simply make the worker rollback any transaction and exit. */ static void rewrite_table_impl(char *relschema_src, char *relname_src, char *relname_new, char *relschema_dst, char *relname_dst) { RangeVar *relrv; Relation rel_src, rel_dst; Oid relid_dst; Oid ident_idx_src; Oid relid_src; Relation ident_index = NULL; ScanKey ident_key; TupleTableSlot *slot_dst_ind = NULL; int i, ident_key_nentries = 0; LogicalDecodingContext *ctx; ReplicationSlot *slot; Snapshot snap_hist; XLogRecPtr end_of_wal; XLogRecPtr xlog_insert_ptr; bool source_finalized; Relation *parts_dst = NULL; int nparts; partitions_hash *partitions = NULL; TupleConversionMapExt *conv_map; EState *estate; ModifyTableState *mtstate; struct PartitionTupleRouting *proute = NULL; List *seqs_src; /* * Use ShareUpdateExclusiveLock as it allows DML commands but does block * most of DDLs (including CREATE INDEX). */ relrv = makeRangeVar(relschema_src, relname_src, -1); rel_src = table_openrv(relrv, ShareUpdateExclusiveLock); relid_src = RelationGetRelid(rel_src); check_prerequisites(rel_src); /* * Retrieve the useful info while holding lock on the relation. */ ident_idx_src = RelationGetReplicaIndex(rel_src); /* The table can have PK although the replica identity is FULL. */ if (ident_idx_src == InvalidOid && rel_src->rd_pkindex != InvalidOid) ident_idx_src = rel_src->rd_pkindex; /* * Check if we're ready to capture changes that possibly take place during * the initial load. * * Note: we let the plugin do this check on per-change basis, and allow * processing of tables with no identity if only INSERT changes are * decoded. However it seems inconsistent. * * XXX Although ERRCODE_UNIQUE_VIOLATION is no actual "unique violation", * this error code seems to be the best match. * (ERRCODE_TRIGGERED_ACTION_EXCEPTION might be worth consideration as * well.) */ if (!OidIsValid(ident_idx_src)) ereport(ERROR, (errcode(ERRCODE_UNIQUE_VIOLATION), (errmsg("Table \"%s\" has no identity index", relname_src)))); /* Prepare for decoding of "concurrent data changes". */ ctx = setup_decoding(rel_src); /* * No one should need to access the destination table during our * processing. We will eventually need AccessExclusiveLock for renaming, * so acquire it right away. * * This should not be done before the call of setup_decoding() as the * exclusive lock does assign XID. (setup_decoding() would then wait for * our transaction to complete.) */ relrv = makeRangeVar(relschema_dst, relname_dst, -1); rel_dst = table_openrv(relrv, AccessExclusiveLock); relid_dst = RelationGetRelid(rel_dst); /* * If the destination table is temporary, user probably messed things up * and a lot of data would be lost at the end of the session. Unlogged * table might be o.k. but let's allow only permanent so far. */ if (!RelationIsPermanent(rel_dst)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a permanent table", relname_dst))); /* * Build a "historic snapshot", i.e. one that reflect the table state at * the moment the snapshot builder reached SNAPBUILD_CONSISTENT state. */ snap_hist = build_historic_snapshot(ctx->snapshot_builder); /* * Cope with commit 706054b11b in PG core. * * If we did this earlier, earlier SnapBuildInitialSnapshot() would raise * ERROR. We shouldn't have called heap_insert|update|delete by now * anyway. */ PushActiveSnapshot(GetTransactionSnapshot()); /* * Create a conversion map so that we can handle difference(s) in the * tuple descriptor. */ conv_map = convert_tuples_by_name_ext(rel_src, rel_dst); /* * Are we going to route the data into partitions? */ if (rel_dst->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) partitions = get_partitions(rel_src, rel_dst, &nparts, &parts_dst, &ident_key, &ident_key_nentries); else { ident_index = get_identity_index(rel_dst, rel_src); ident_key = build_identity_key(ident_index, &ident_key_nentries); slot_dst_ind = table_slot_create(rel_dst, NULL); } Assert(ident_key_nentries > 0); /* Executor state to determine the target partition. */ estate = CreateExecutorState(); /* * XXX Is it ok to leave the CMD_INSERT command for processing of the * concurrent changes, which includes UPDATE and DELETE commands? */ mtstate = get_modify_table_state(estate, rel_dst, CMD_INSERT); if (partitions) { #if PG_VERSION_NUM >= 140000 proute = ExecSetupPartitionTupleRouting(estate, rel_dst); #else proute = ExecSetupPartitionTupleRouting(estate, mtstate, rel_dst); #endif } /* * The historic snapshot is used to retrieve data w/o concurrent changes. */ snap_hist = RegisterSnapshot(snap_hist); perform_initial_load(estate, mtstate, proute, rel_src, snap_hist, rel_dst, partitions, ctx, conv_map); UnregisterSnapshot(snap_hist); /* * We no longer need to preserve the rows processed during the initial * load from VACUUM. (User is not likely to run VACUUM on a table that we * currently process, but our stale effective_xmin would also restrict * VACUUM on other tables.) */ slot = ctx->slot; SpinLockAcquire(&slot->mutex); Assert(TransactionIdIsValid(slot->effective_xmin) && !TransactionIdIsValid(slot->data.xmin)); slot->effective_xmin = InvalidTransactionId; SpinLockRelease(&slot->mutex); /* * This is rather paranoia than anything else --- perform_initial_load() * uses each snapshot to access different table, and it does not cause * catalog changes. */ InvalidateSystemCaches(); /* * Make sure the contents of the destination partitions is visible, for * the sake of concurrent data changes. */ CommandCounterIncrement(); /* * During testing, wait for another backend to perform concurrent data * changes which we will process below. */ #if PG_VERSION_NUM >= 180000 INJECTION_POINT("pg_rewrite-before-lock", NULL); #elif PG_VERSION_NUM >= 170000 INJECTION_POINT("pg_rewrite-before-lock"); #endif /* * Flush all WAL records inserted so far (possibly except for the last * incomplete page, see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ xlog_insert_ptr = GetInsertRecPtr(); XLogFlush(xlog_insert_ptr); /* * Since we'll do some more changes, all the WAL records flushed so far * need to be decoded for sure. */ #if PG_VERSION_NUM >= 150000 end_of_wal = GetFlushRecPtr(NULL); #else end_of_wal = GetFlushRecPtr(); #endif /* * Decode and apply the data changes that occurred while the initial load * was in progress. The XLOG reader should continue where setup_decoding() * has left it. * * Even if the amount of concurrent changes of our source table might not * be significant, both initial load and index build could have produced * many XLOG records that we need to read. Do so before requesting * exclusive lock on the source relation. */ pg_rewrite_process_concurrent_changes(estate, mtstate, proute, ctx, end_of_wal, ident_key, ident_key_nentries, ident_index, slot_dst_ind, NoLock, partitions, conv_map, NULL); /* * Try a few times to perform the stage that requires exclusive lock on * the source relation. * * XXX Not sure the number of attempts should be configurable. If it fails * several times, admin should either increase partition_max_xlock_time or * disable it. */ source_finalized = false; for (i = 0; i < 4; i++) { if (perform_final_merge(estate, mtstate, proute, rel_src, ident_key, ident_key_nentries, ident_index, slot_dst_ind, ctx, partitions, conv_map)) { source_finalized = true; break; } else elog(DEBUG1, "pg_rewrite: exclusive lock on table %u had to be released.", relid_src); } if (!source_finalized) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("pg_rewrite: \"max_xlock_time\" prevented partitioning from completion"))); /* * Retrieve information on sequences so that we can eventually set them on * rel_dst. */ seqs_src = get_sequences(rel_src); /* rel_src cache entry is not needed anymore, but the lock is. */ table_close(rel_src, NoLock); /* * Done with decoding. * * XXX decoding_cleanup() frees tup_desc_src, although we've used it not * only for the decoding. */ decoding_cleanup(ctx); ReplicationSlotRelease(); pfree(ident_key); /* * Besides explicitly closing rel_dst, make sure it (and possibly its * partitions) is not referenced indirectly copy_constraints() below runs * ALTER TABLE which in turn does not like leftover relcache references. */ if (proute) { ExecCleanupTupleRouting(mtstate, proute); pfree(proute); } if (estate->es_partition_directory) { DestroyPartitionDirectory(estate->es_partition_directory); estate->es_partition_directory = NULL; } if (partitions) { close_partitions(partitions); for (i = 0; i < nparts; i++) table_close(parts_dst[i], AccessExclusiveLock); pfree(parts_dst); } /* * If the source table had sequences, apply their values to the * corresponding sequences of the destination table. */ if (seqs_src) { set_sequences(rel_dst, seqs_src); list_free_deep(seqs_src); } /* * The relcache reference is no longer needed, so close it. Unlocking will * will take place at the end of transaction. * * Note that RenameRelationInternal(relid_dst, ...) below will lock the * relation using AccessExclusiveLock mode once more. This lock will also * be released at the end of our transaction. */ table_close(rel_dst, NoLock); /* * Rename the source table so that we can reuse its name (relname_src) * below. * * The lock acquired by perform_final_merge() will be released at the end * of transaction - no need to deal with it here. (The same applies to the * lock acquired by this call.) */ RenameRelationInternal(relid_src, relname_new, false, false); /* * The relation we have just populated will be renamed so it replaces the * original one. Before that, make sure that the previous renaming is * visible so that we can reuse relname_src. */ CommandCounterIncrement(); /* * Finally, rename the newly populated relation so it replaces the * original one. * * We have AccessExclusiveLock lock on relid_dst since we opened it first * time and it will be released at the end of transaction (The same * applies to the lock acquired by this call.) */ RenameRelationInternal(relid_dst, relname_src, false, false); /* * Create FK and CHECK constraints on rel_dst (renamed now to relname_src) * according to rel_src (renamed now to relname_new), and mark them NOT * VALID. See the comments of copy_constraints() for details. */ copy_constraints(relid_dst, relname_src, relid_src); /* Cleanup */ if (partitions == NULL) { index_close(ident_index, AccessShareLock); ExecDropSingleTupleTableSlot(slot_dst_ind); } free_modify_table_state(mtstate); /* * ExecFindPartition() might have pinned tuple descriptors of the * partitions. */ ExecResetTupleTable(estate->es_tupleTable, true); FreeExecutorState(estate); if (conv_map) free_conversion_map_ext(conv_map); /* See PushActiveSnapshot() above. */ PopActiveSnapshot(); } /* * Check that both relations have matching identity indexes and return the * identity index of 'rel_dst'. */ static Relation get_identity_index(Relation rel_dst, Relation rel_src) { Oid index_dst_oid, index_src_oid; Relation index_dst, index_src; TupleDesc tupdesc_dst, tupdesc_src; bool match = true; index_dst_oid = RelationGetReplicaIndex(rel_dst); if (!OidIsValid(index_dst_oid)) elog(ERROR, "Identity index missing on table \"%s\"", RelationGetRelationName(rel_dst)); index_dst = index_open(index_dst_oid, AccessShareLock); tupdesc_dst = RelationGetDescr(index_dst); index_src_oid = RelationGetReplicaIndex(rel_src); if (!OidIsValid(index_src_oid)) elog(ERROR, "Identity index missing on table \"%s\"", RelationGetRelationName(rel_src)); index_src = index_open(index_src_oid, AccessShareLock); tupdesc_src = RelationGetDescr(index_src); /* * The tuple descriptors might not be equal, since some attributes can * have different types. What should match though is attribute names and * their order. */ if (tupdesc_src->natts != tupdesc_dst->natts) match = false; else { for (int i = 0; i < tupdesc_src->natts; i++) { Form_pg_attribute att_src = TupleDescAttr(tupdesc_src, i); Form_pg_attribute att_dst = TupleDescAttr(tupdesc_dst, i); /* Indexes should not have dropped attributes. */ Assert(!att_src->attisdropped); Assert(!att_dst->attisdropped); if (strcmp(NameStr(att_src->attname), NameStr(att_dst->attname)) != 0) { match = false; break; } } } if (!match) elog(ERROR, "identity index on table \"%s\" does not match that on table \"%s\"", RelationGetRelationName(rel_dst), RelationGetRelationName(rel_src)); index_close(index_src, AccessShareLock); return index_dst; } /* * Retrieve information needed to apply DML commands to partitioned table. */ static partitions_hash * get_partitions(Relation rel_src, Relation rel_dst, int *nparts, Relation **parts_dst_p, ScanKey *ident_key_p, int *ident_key_nentries) { partitions_hash *partitions; Relation *parts_dst; ScanKey ident_key = NULL; PartitionDesc part_desc; #if PG_VERSION_NUM >= 140000 part_desc = RelationGetPartitionDesc(rel_dst, true); #else part_desc = RelationGetPartitionDesc(rel_dst); #endif if (part_desc->nparts == 0) ereport(ERROR, (errmsg("table \"%s\" has no partitions", RelationGetRelationName(rel_dst)))); /* * It's probably not necessary to lock the partitions in exclusive mode, * but we'll need to open them later. Simply use the exclusive lock * instead of trying to determine the minimum lock level needed. */ parts_dst = (Relation *) palloc(part_desc->nparts * sizeof(Relation)); for (int i = 0; i < part_desc->nparts; i++) parts_dst[i] = table_open(part_desc->oids[i], AccessExclusiveLock); /* * Pointers to identity indexes will be looked up by the partition * relation OID. */ partitions = partitions_create(CurrentMemoryContext, 8, NULL); /* * Gather partition information that we'll need later. It happens here * because it's a good opportunity to check the partition tuple * descriptors and identity indexes before the initial load starts. (The * load does not need those indexes, but it'd be unfortunate to find out * incorrect or missing identity index after the initial load has been * performed.) */ for (int i = 0; i < part_desc->nparts; i++) { Relation partition = parts_dst[i]; PartitionEntry *entry; bool found; /* Info on partitions. */ entry = partitions_insert(partitions, RelationGetRelid(partition), &found); Assert(!found); /* * Identity of the rows of a foreign table is hard to implement for * foreign tables. We'd hit the problem below, but it's clearer to * report the problem this way. */ if (partition->rd_rel->relkind == RELKIND_FOREIGN_TABLE) ereport(ERROR, (errmsg("\"%s\" is a foreign table", RelationGetRelationName(partition)))); entry->ident_index = get_identity_index(partition, rel_src); entry->slot_ind = table_slot_create(partition, NULL); entry->slot = MakeSingleTupleTableSlot(RelationGetDescr(rel_dst), &TTSOpsHeapTuple); entry->conv_map = convert_tuples_by_name_ext(rel_dst, partition); /* Expect many insertions. */ entry->bistate = GetBulkInsertState(); /* * Build scan key that we'll use to look for rows to be updated / * deleted during logical decoding. * * As all the partitions have the same definition of the identity * index, there should only be a single identity key. */ if (ident_key == NULL) { ident_key = build_identity_key(entry->ident_index, ident_key_nentries); *ident_key_p = ident_key; } } *nparts = part_desc->nparts; *parts_dst_p = parts_dst; return partitions; } /* * Return a list of SequenceValue for given relation. */ static List * get_sequences(Relation rel) { Oid foid; FmgrInfo flinfo; TupleDesc tupdesc; List *result = NIL; bool skipped = false; foid = fmgr_internal_function("pg_sequence_last_value"); Assert(OidIsValid(foid)); fmgr_info(foid, &flinfo); tupdesc = RelationGetDescr(rel); for (int i = 0; i < tupdesc->natts; i++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, i); List *seqlist; Oid seqid; LOCAL_FCINFO(fcinfo, 1); Datum last_value; Assert(attr->attnum == (i + 1)); if (attr->attisdropped) continue; seqlist = getOwnedSequences_internal(RelationGetRelid(rel), attr->attnum, 0); /* * Obviously do nothing if there is no sequence for the attribute. If * there are more then one, ignore that attribute too. The latter * probably should not happen, but if it does, we issue a log message * rather than aborting the whole rewrite. */ if (list_length(seqlist) != 1) { if (list_length(seqlist) > 1) skipped = true; continue; } seqid = linitial_oid(seqlist); /* * FunctionCall1() cannot be used here because the function we call * can return NULL. */ InitFunctionCallInfoData(*fcinfo, &flinfo, 1, InvalidOid, NULL, NULL); fcinfo->args[0].value = ObjectIdGetDatum(seqid); fcinfo->args[0].isnull = false; last_value = FunctionCallInvoke(fcinfo); if (!fcinfo->args[0].isnull) { SequenceValue *sv = palloc_object(SequenceValue); sv->attname = attr->attname; sv->last_value = DatumGetInt64(last_value); result = lappend(result, sv); } } if (skipped) send_message(MyWorkerTask, NOTICE, "could not get sequence value(s) from the source table", NULL); return result; } /* * Copy & pasted from PG core. The problem is that we need to call the * function with attnum > 0 and deptype = 0. PG core does not expose function * that would do exactly that. */ static List * getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) { List *result = NIL; Relation depRel; ScanKeyData key[3]; SysScanDesc scan; HeapTuple tup; depRel = table_open(DependRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); if (attnum) ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attnum)); scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, attnum ? 3 : 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); /* * We assume any auto or internal dependency of a sequence on a column * must be what we are looking for. (We need the relkind test because * indexes can also have auto dependencies on columns.) */ if (deprec->classid == RelationRelationId && deprec->objsubid == 0 && deprec->refobjsubid != 0 && (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) { if (!deptype || deprec->deptype == deptype) result = lappend_oid(result, deprec->objid); } } systable_endscan(scan); table_close(depRel, AccessShareLock); return result; } /* * Set sequences according to the information retrieved by get_sequences(). */ static void set_sequences(Relation rel, List *seqs_src) { Oid relid = RelationGetRelid(rel); Oid foid; FmgrInfo flinfo; ListCell *lc; bool skipped = false; foid = fmgr_internal_function("setval_oid"); Assert(OidIsValid(foid)); fmgr_info(foid, &flinfo); foreach(lc, seqs_src) { SequenceValue *sv = (SequenceValue *) lfirst(lc); AttrNumber attnum; List *seqlist; Oid seqid; attnum = get_attnum(relid, NameStr(sv->attname)); seqlist = getOwnedSequences_internal(relid, attnum, 0); /* * Unlike get_sequences(), here we have a problem even if there is no * sequence on the attribute. (Because we know that the sequence * exists on the source table.) */ if (list_length(seqlist) != 1) { skipped = true; continue; } seqid = linitial_oid(seqlist); FunctionCall2(&flinfo, ObjectIdGetDatum(seqid), Int64GetDatum(sv->last_value)); } if (skipped) send_message(MyWorkerTask, NOTICE, "could not identify sequence(s) on the target table", NULL); } /* * Raise error if the relation is not eligible for partitioning or any adverse * conditions exist. * * Some of the checks may be redundant (e.g. heap_open() checks relkind) but * its safer to have them all listed here. */ static void check_prerequisites(Relation rel) { Form_pg_class form = RelationGetForm(rel); /* Check the relation first. */ if (form->relkind == RELKIND_PARTITIONED_TABLE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("the source table may not be partitioned"))); if (form->relkind != RELKIND_RELATION) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a table", RelationGetRelationName(rel)))); if (!RelationIsPermanent(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a permanent table", RelationGetRelationName(rel)))); if (form->relisshared) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is shared relation", RelationGetRelationName(rel)))); if (RelationIsMapped(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is mapped relation", RelationGetRelationName(rel)))); /* * There's no urgent need to process catalog tables. * * Should this limitation be relaxed someday, consider if we need to write * xl_heap_rewrite_mapping records. (Probably not because the whole * "decoding session" takes place within a call of partition_table() and * our catalog checks should not allow for a concurrent rewrite that could * make snapmgr.c:tuplecid_data obsolete. Furthermore, such a rewrite * would have to take place before perform_initial_load(), but this is * called before any transactions could have been decoded, so tuplecid * should still be empty anyway.) */ if (RelationGetRelid(rel) < FirstNormalObjectId) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not user relation", RelationGetRelationName(rel)))); /* * While AFTER trigger should not be an issue (to generate an event must * have got XID assigned, causing setup_decoding() to fail later), open * cursor might be. See comments of the function for details. */ CheckTableNotInUse(rel, "rewrite_table()"); } /* * This function is much like pg_create_logical_replication_slot() except that * the new slot is neither released (if anyone else could read changes from * our slot, we could miss changes other backends do while we copy the * existing data into temporary table), nor persisted (it's easier to handle * crash by restarting all the work from scratch). * * XXX Even though CreateInitDecodingContext() does not set state to * RS_PERSISTENT, it does write the slot to disk. We rely on * RestoreSlotFromDisk() to delete ephemeral slots during startup. (Both ERROR * and FATAL should lead to cleanup even before the cluster goes down.) */ static LogicalDecodingContext * setup_decoding(Relation rel) { Oid relid = RelationGetRelid(rel); StringInfo buf; LogicalDecodingContext *ctx; DecodingOutputState *dstate; MemoryContext oldcontext; /* check_permissions() "inlined", as logicalfuncs.c does not export it. */ if (!superuser() && !has_rolreplication(GetUserId())) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser or replication role to use replication slots")))); CheckLogicalDecodingRequirements(); /* Make sure there's no conflict with the SPI and its contexts. */ oldcontext = MemoryContextSwitchTo(TopTransactionContext); /* * In order to be able to run partition_table() for multiple tables at a * time, slot name should contain both database OID and relation OID. */ buf = makeStringInfo(); appendStringInfoString(buf, REPL_SLOT_BASE_NAME); appendStringInfo(buf, "%u_%u", MyDatabaseId, relid); #if PG_VERSION_NUM >= 170000 ReplicationSlotCreate(buf->data, true, RS_EPHEMERAL, false, false, false); #elif PG_VERSION_NUM >= 140000 ReplicationSlotCreate(buf->data, true, RS_EPHEMERAL, false); #else ReplicationSlotCreate(buf->data, true, RS_EPHEMERAL); #endif /* * Neither prepare_write nor do_write callback nor update_progress is * useful for us. * * Regarding the value of need_full_snapshot, we pass true to protect its * data from VACUUM. Otherwise the historical snapshot we use for the * initial load could miss some data. (Unlike logical decoding, we need * the historical snapshot for non-catalog tables.) */ ctx = CreateInitDecodingContext(REPL_PLUGIN_NAME, NIL, true, InvalidXLogRecPtr, XL_ROUTINE(.page_read = read_local_xlog_page, .segment_open = wal_segment_open, .segment_close = wal_segment_close), NULL, NULL, NULL); /* * We don't have control on setting fast_forward, so at least check it. */ Assert(!ctx->fast_forward); DecodingContextFindStartpoint(ctx); /* Some WAL records should have been read. */ Assert(ctx->reader->EndRecPtr != InvalidXLogRecPtr); XLByteToSeg(ctx->reader->EndRecPtr, rewrite_current_segment, wal_segment_size); /* * Setup structures to store decoded changes. */ dstate = palloc0(sizeof(DecodingOutputState)); dstate->relid = relid; dstate->tstore = tuplestore_begin_heap(false, false, maintenance_work_mem); /* Initialize the descriptor to store the changes ... */ dstate->tupdesc_change = CreateTemplateTupleDesc(1); TupleDescInitEntry(dstate->tupdesc_change, 1, NULL, BYTEAOID, -1, 0); /* ... as well as the corresponding slot. */ dstate->tsslot = MakeSingleTupleTableSlot(dstate->tupdesc_change, &TTSOpsMinimalTuple); dstate->resowner = ResourceOwnerCreate(CurrentResourceOwner, "logical decoding"); /* * Tuple descriptor of the source relation might be needed for decoding. */ dstate->tupdesc_src = RelationGetDescr(rel); MemoryContextSwitchTo(oldcontext); ctx->output_writer_private = dstate; return ctx; } static void decoding_cleanup(LogicalDecodingContext *ctx) { DecodingOutputState *dstate; dstate = (DecodingOutputState *) ctx->output_writer_private; ExecDropSingleTupleTableSlot(dstate->tsslot); FreeTupleDesc(dstate->tupdesc_change); tuplestore_end(dstate->tstore); FreeDecodingContext(ctx); } /* * Create ModifyTableState and do the minimal initialization so that * ExecFindPartition() works. */ static ModifyTableState * get_modify_table_state(EState *estate, Relation rel, CmdType operation) { ModifyTableState *result = makeNode(ModifyTableState); ResultRelInfo *rri = makeNode(ResultRelInfo); InitResultRelInfo(rri, rel, 0, NULL, 0); ExecOpenIndices(rri, false); result->ps.plan = NULL; result->ps.state = estate; result->operation = operation; #if PG_VERSION_NUM >= 140000 result->mt_nrels = 1; #endif result->resultRelInfo = rri; result->rootResultRelInfo = rri; return result; } static void free_modify_table_state(ModifyTableState *mtstate) { ExecCloseIndices(mtstate->resultRelInfo); pfree(mtstate->resultRelInfo); pfree(mtstate); } /* * Wrapper for SnapBuildInitialSnapshot(). * * We do not have to meet the assertions that SnapBuildInitialSnapshot() * contains, nor should we set MyPgXact->xmin. */ static Snapshot build_historic_snapshot(SnapBuild *builder) { Snapshot result; bool FirstSnapshotSet_save; int XactIsoLevel_save; TransactionId xmin_save; /* * Fake both FirstSnapshotSet and XactIsoLevel so that the assertions in * SnapBuildInitialSnapshot() don't fire. Otherwise partition_table() has * no reason to apply these values. */ FirstSnapshotSet_save = FirstSnapshotSet; FirstSnapshotSet = false; XactIsoLevel_save = XactIsoLevel; XactIsoLevel = XACT_REPEATABLE_READ; /* * Likewise, fake MyPgXact->xmin so that the corresponding check passes. */ #if PG_VERSION_NUM >= 140000 xmin_save = MyProc->xmin; MyProc->xmin = InvalidTransactionId; #else xmin_save = MyPgXact->xmin; MyPgXact->xmin = InvalidTransactionId; #endif /* * Call the core function to actually build the snapshot. */ result = SnapBuildInitialSnapshot(builder); /* * Restore the original values. */ FirstSnapshotSet = FirstSnapshotSet_save; XactIsoLevel = XactIsoLevel_save; #if PG_VERSION_NUM >= 140000 MyProc->xmin = xmin_save; #else MyPgXact->xmin = xmin_save; #endif return result; } /* * Use snap_hist snapshot to get the relevant data from rel_src and insert it * into the appropriate partitions of rel_dst. * * Caller is responsible for opening and locking the source relation. */ static void perform_initial_load(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, Relation rel_src, Snapshot snap_hist, Relation rel_dst, partitions_hash *partitions, LogicalDecodingContext *ctx, TupleConversionMapExt *conv_map) { int batch_size, batch_max_size; Size tuple_array_size; bool tuple_array_can_expand = true; TableScanDesc heap_scan; TupleTableSlot *slot_src, *slot_dst; HeapTuple *tuples = NULL; BulkInsertState bistate_nonpart = NULL; MemoryContext load_cxt, old_cxt; XLogRecPtr end_of_wal_prev = InvalidXLogRecPtr; DecodingOutputState *dstate; char replorigin_name[255]; /* * The session origin will be used to mark WAL records produced by the * load itself so that they are not decoded. */ Assert(replorigin_session_origin == InvalidRepOriginId); snprintf(replorigin_name, sizeof(replorigin_name), REPLORIGIN_NAME_PATTERN, MyDatabaseId); replorigin_session_origin = replorigin_create(replorigin_name); /* * Also remember that the WAL records created during the load should not * be decoded later. */ dstate = (DecodingOutputState *) ctx->output_writer_private; dstate->rorigin = replorigin_session_origin; heap_scan = table_beginscan(rel_src, snap_hist, 0, (ScanKey) NULL); /* Slot to retrieve data from the source table. */ slot_src = table_slot_create(rel_src, NULL); /* * Slot to be passed to ExecFindPartition(). We use a separate slot * because here we want to enforce the TTSOpsHeapTuple because the tuple * we pass to ExecFindPartition() is a HeapTuple for sure (i.e we try to * avoid deforming the tuple when storing it). */ slot_dst = MakeSingleTupleTableSlot(RelationGetDescr(rel_dst), &TTSOpsHeapTuple); /* * If the table is partitioned, each partition has a separate instance of * BulkInsertState. Otherwise we need to allocate one here. */ if (proute == NULL) bistate_nonpart = GetBulkInsertState(); /* * Store as much data as we can store in memory. The more memory is * available, the fewer iterations. */ batch_max_size = 1024; tuple_array_size = batch_max_size * sizeof(HeapTuple); /* The minimum value of maintenance_work_mem is 1024 kB. */ Assert(tuple_array_size / 1024 < maintenance_work_mem); tuples = (HeapTuple *) palloc(tuple_array_size); /* * The processing can take many iterations. In case any data manipulation * below leaked, try to defend against out-of-memory conditions by using a * separate memory context. */ load_cxt = AllocSetContextCreate(CurrentMemoryContext, "pg_rewrite initial load cxt", ALLOCSET_DEFAULT_SIZES); old_cxt = MemoryContextSwitchTo(load_cxt); while (true) { HeapTuple tup_in = NULL; int i; Size data_size = 0; XLogRecPtr end_of_wal; for (i = 0;; i++) { bool flattened = false; /* * Check if the tuple array fits into maintenance_work_mem. * * Since the tuple cannot be put back to the scan, it'd make * things tricky if we involved the current tuple in the * computation. Since the unit of maintenance_work_mem is kB, one * extra tuple shouldn't hurt too much. */ if (((data_size + tuple_array_size) / 1024) >= maintenance_work_mem) { /* * data_size should still be zero if tup_in is the first item * of the current batch and the array itself should never * exceed maintenance_work_mem. XXX If the condition above is * changed to include the current tuple (i.e. we put the * current tuple aside for the next batch), make sure the * first tuple of a batch is inserted regardless its size. We * cannot shrink the array in favor of actual data in generic * case (i.e. tuple size can in general be bigger than * maintenance_work_mem). */ Assert(i > 0); break; } /* * Perform the tuple retrieval in the original context so that no * scan data is freed during the cleanup between batches. */ MemoryContextSwitchTo(old_cxt); { bool res; res = table_scan_getnextslot(heap_scan, ForwardScanDirection, slot_src); if (res) { bool shouldFree; tup_in = ExecFetchSlotHeapTuple(slot_src, false, &shouldFree); /* TTSOpsBufferHeapTuple has .get_heap_tuple != NULL. */ Assert(!shouldFree); } else tup_in = NULL; } MemoryContextSwitchTo(load_cxt); /* * Ran out of input data? */ if (tup_in == NULL) break; /* * Even though special snapshot is used to retrieve values from * TOAST relation (see toast_fetch_datum), we'd better flatten the * tuple and thus retrieve the TOAST while the historic snapshot * is active. One particular reason is that tuptoaster.c does * access catalog. */ if (HeapTupleHasExternal(tup_in)) { tup_in = toast_flatten_tuple(tup_in, RelationGetDescr(rel_src)); flattened = true; } pg_rewrite_exit_if_requested(); /* * Check for a free slot early enough so that the current tuple * can be stored even if the array cannot be reallocated. Do not * try again and again if the tuple array reached the maximum * value. */ if (i == (batch_max_size - 1) && tuple_array_can_expand) { int batch_max_size_new; Size tuple_array_size_new; batch_max_size_new = 2 * batch_max_size; tuple_array_size_new = batch_max_size_new * sizeof(HeapTuple); /* * Besides being of valid size, the new array should allow for * storing some data w/o exceeding maintenance_work_mem. XXX * Consider tuning the portion of maintenance_work_mem that * the array can use. */ if (!AllocSizeIsValid(tuple_array_size_new) || tuple_array_size_new / 1024 >= maintenance_work_mem / 16) tuple_array_can_expand = false; /* * Only expand the array if the current iteration does not * violate maintenance_work_mem. */ if (tuple_array_can_expand) { tuples = (HeapTuple *) repalloc(tuples, tuple_array_size_new); batch_max_size = batch_max_size_new; tuple_array_size = tuple_array_size_new; } } if (!flattened) tup_in = heap_copytuple(tup_in); /* Store the tuple and account for its size. */ tuples[i] = tup_in; data_size += HEAPTUPLESIZE + tup_in->t_len; /* * If the tuple array could not be expanded, stop reading for the * current batch. */ if (i == (batch_max_size - 1)) { /* The current tuple belongs to the current batch. */ i++; break; } } /* * Insert the tuples into the target table. * * pg_rewrite_check_catalog_changes() shouldn't be necessary as long * as the AccessSqhareLock we hold on the source relation does not * allow change of table type. (Should ALTER INDEX take place * concurrently, it does not break the heap insertions. In such a case * we'll find out later that we need to terminate processing of the * current table, but it's probably not worth checking each batch.) */ /* * Has the previous batch processed all the remaining tuples? * * In theory, the counter might end up zero as a result of overflow. * However in practice 'i' should not overflow because its upper limit * is controlled by 'batch_max_size' which is also of the int data * type, and which in turn should not overflow because value much * lower than INT_MAX will make AllocSizeIsValid(tuple_array_size_new) * return false. */ if (i == 0) break; batch_size = i; i = 0; while (true) { HeapTuple tup_out; ResultRelInfo *rri; Relation rel_ins; List *recheck; BulkInsertState bistate; pg_rewrite_exit_if_requested(); if (i == batch_size) tup_out = NULL; else tup_out = tuples[i++]; if (tup_out == NULL) break; /* * If needed, convert the tuple so it matches the destination * table. */ if (conv_map) tup_out = convert_tuple_for_dest_table(tup_out, conv_map); ExecStoreHeapTuple(tup_out, slot_dst, false); if (proute) { PartitionEntry *entry; /* Find out which partition the tuple belongs to. */ rri = ExecFindPartition(mtstate, mtstate->rootResultRelInfo, proute, slot_dst, estate); rel_ins = rri->ri_RelationDesc; entry = get_partition_entry(partitions, RelationGetRelid(rri->ri_RelationDesc)); bistate = entry->bistate; /* * Make sure the tuple matches the partition. The typical * problem we address here is that a partition was attached * that has a different order of columns. */ if (entry->conv_map) { tup_out = convert_tuple_for_dest_table(tup_out, entry->conv_map); ExecClearTuple(slot_dst); ExecStoreHeapTuple(tup_out, slot_dst, false); } } else { /* Non-partitioned table. */ rri = mtstate->resultRelInfo; rel_ins = rel_dst; bistate = bistate_nonpart; } /* * Insert the tuple into the relation (or partition). * * XXX Should this happen outside load_cxt? Currently "bistate" is * a flat object (i.e. it does not point to any memory chunk that * the previous call of table_tuple_insert() might have allocated) * and thus the cleanup between batches should not damage it, but * can't it get more complex in future PG versions? */ Assert(bistate != NULL); table_tuple_insert(rel_ins, slot_dst, GetCurrentCommandId(true), 0, bistate); #if PG_VERSION_NUM < 140000 estate->es_result_relation_info = rri; #endif /* Update indexes. */ recheck = ExecInsertIndexTuples( #if PG_VERSION_NUM >= 140000 rri, #endif slot_dst, estate, #if PG_VERSION_NUM >= 140000 false, /* update */ #endif false, /* noDupErr */ NULL, /* specConflict */ NIL /* arbiterIndexes */ #if PG_VERSION_NUM >= 160000 , false /* onlySummarizing */ #endif ); ExecClearTuple(slot_dst); /* * If recheck is required, it must have been preformed on the * source relation by now. (All the logical changes we process * here are already committed.) */ list_free(recheck); pfree(tup_out); /* Update the progress information. */ SpinLockAcquire(&MyWorkerTask->mutex); MyWorkerTask->progress.ins_initial++; SpinLockRelease(&MyWorkerTask->mutex); } /* * Reached the end of scan when retrieving data from the source table? */ if (tup_in == NULL) break; /* * Free possibly-leaked memory. */ MemoryContextReset(load_cxt); /* * Decode the WAL produced by the load, as well as by other * transactions, so that the replication slot can advance and WAL does * not pile up. Of course we must not apply the changes until the * initial load has completed. * * Note that the insertions into the new table shouldn't actually be * decoded, they should be filtered out by their origin. */ #if PG_VERSION_NUM >= 150000 end_of_wal = GetFlushRecPtr(NULL); #else end_of_wal = GetFlushRecPtr(); #endif if (end_of_wal > end_of_wal_prev) { MemoryContextSwitchTo(old_cxt); pg_rewrite_decode_concurrent_changes(ctx, end_of_wal, NULL); MemoryContextSwitchTo(load_cxt); } end_of_wal_prev = end_of_wal; } /* * At whichever stage the loop broke, the historic snapshot should no * longer be active. */ /* Cleanup. */ pfree(tuples); table_endscan(heap_scan); ExecDropSingleTupleTableSlot(slot_src); ExecDropSingleTupleTableSlot(slot_dst); if (bistate_nonpart) FreeBulkInsertState(bistate_nonpart); /* Drop the replication origin. */ #if PG_VERSION_NUM >= 140000 replorigin_drop_by_name(replorigin_name, false, true); #else replorigin_drop(replorigin_session_origin, false); #endif replorigin_session_origin = InvalidRepOriginId; MemoryContextSwitchTo(old_cxt); MemoryContextDelete(load_cxt); elog(DEBUG1, "pg_rewrite: the initial load completed"); } /* * Build scan key to process logical changes. */ static ScanKey build_identity_key(Relation ident_idx_rel, int *nentries) { HeapTuple ht_idx; Datum coldatum; bool colisnull; oidvector *indcollation; int n, i; ScanKey result; ht_idx = ident_idx_rel->rd_indextuple; coldatum = SysCacheGetAttr(INDEXRELID, ht_idx, Anum_pg_index_indcollation, &colisnull); Assert(!colisnull); indcollation = (oidvector *) DatumGetPointer(coldatum); n = RelationGetNumberOfAttributes(ident_idx_rel); result = (ScanKey) palloc(sizeof(ScanKeyData) * n); for (i = 0; i < n; i++) { ScanKey entry; Oid opfamily, opcintype, opno, opcode; entry = &result[i]; opfamily = ident_idx_rel->rd_opfamily[i]; opcintype = ident_idx_rel->rd_opcintype[i]; opno = get_opfamily_member(opfamily, opcintype, opcintype, BTEqualStrategyNumber); if (!OidIsValid(opno)) elog(ERROR, "Failed to find = operator for type %u", opcintype); opcode = get_opcode(opno); if (!OidIsValid(opcode)) elog(ERROR, "Failed to find = operator for operator %u", opno); /* Initialize everything but argument. */ ScanKeyInit(entry, i + 1, BTEqualStrategyNumber, opcode, (Datum) NULL); entry->sk_collation = indcollation->values[i]; } *nentries = n; return result; } /* * Try to perform the final processing of concurrent data changes of the * source table, which requires an exclusive lock. The return value tells * whether this step succeeded. (If not, caller might want to retry.) */ static bool perform_final_merge(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, Relation rel_src, ScanKey ident_key, int ident_key_nentries, Relation ident_index, TupleTableSlot *slot_dst_ind, LogicalDecodingContext *ctx, partitions_hash *partitions, TupleConversionMapExt *conv_map) { bool success; XLogRecPtr xlog_insert_ptr, end_of_wal; struct timeval t_end; struct timeval *t_end_ptr = NULL; char dummy_rec_data = '\0'; List *indexes; ListCell *lc; /* * Lock the source table exclusively, to finalize the work. */ LockRelationOid(RelationGetRelid(rel_src), AccessExclusiveLock); /* * Lock the indexes too, as ALTER INDEX does not need table lock. */ indexes = RelationGetIndexList(rel_src); foreach(lc, indexes) LockRelationOid(lfirst_oid(lc), AccessExclusiveLock); if (rewrite_max_xlock_time > 0) { int64 usec; struct timeval t_start; int max_xlock_time = MyWorkerTask->max_xlock_time; gettimeofday(&t_start, NULL); /* Add the whole seconds. */ t_end.tv_sec = t_start.tv_sec + max_xlock_time / 1000; /* Add the rest, expressed in microseconds. */ usec = t_start.tv_usec + 1000 * (max_xlock_time % 1000); /* The number of microseconds could have overflown. */ t_end.tv_sec += usec / USECS_PER_SEC; t_end.tv_usec = usec % USECS_PER_SEC; t_end_ptr = &t_end; elog(DEBUG1, "pg_rewrite: completion required by %lu.%lu, current time is %lu.%lu.", t_end_ptr->tv_sec, t_end_ptr->tv_usec, t_start.tv_sec, t_start.tv_usec); } /* * Flush anything we see in WAL, to make sure that all changes committed * while we were creating indexes and waiting for the exclusive lock are * available for decoding. This should not be necessary if all backends * had synchronous_commit set, but we can't rely on this setting. * * Unfortunately, GetInsertRecPtr() may lag behind the actual insert * position, and GetLastImportantRecPtr() points at the start of the last * record rather than at the end. Thus the simplest way to determine the * insert position is to insert a dummy record and use its LSN. * * XXX Consider using GetLastImportantRecPtr() and adding the size of the * last record (plus the total size of all the page headers the record * spans)? */ XLogBeginInsert(); XLogRegisterData(&dummy_rec_data, 1); xlog_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); XLogFlush(xlog_insert_ptr); #if PG_VERSION_NUM >= 150000 end_of_wal = GetFlushRecPtr(NULL); #else end_of_wal = GetFlushRecPtr(); #endif /* * Process the changes that might have taken place while we were waiting * for the lock. * * AccessExclusiveLock effectively disables catalog checks - we've already * performed them above. */ success = pg_rewrite_process_concurrent_changes(estate, mtstate, proute, ctx, end_of_wal, ident_key, ident_key_nentries, ident_index, slot_dst_ind, AccessExclusiveLock, partitions, conv_map, t_end_ptr); if (t_end_ptr) { struct timeval t_now; gettimeofday(&t_now, NULL); elog(DEBUG1, "pg_rewrite: concurrent changes processed at %lu.%lu, result: %u", t_now.tv_sec, t_now.tv_usec, success); } if (!success) { /* Unlock the relation and indexes. */ UnlockRelationOid(RelationGetRelid(rel_src), AccessExclusiveLock); foreach(lc, indexes) UnlockRelationOid(lfirst_oid(lc), AccessExclusiveLock); /* * Take time to reach end_of_wal. * * XXX DecodingOutputState may contain some changes. The corner case * that the data_size has already reached maintenance_work_mem so the * first change we decode now will make it spill to disk is too low to * justify calling apply_concurrent_changes() separately. */ pg_rewrite_process_concurrent_changes(estate, mtstate, proute, ctx, end_of_wal, ident_key, ident_key_nentries, ident_index, slot_dst_ind, AccessExclusiveLock, partitions, conv_map, NULL); /* No time constraint, all changes must have been processed. */ Assert(((DecodingOutputState *) ctx->output_writer_private)->nchanges == 0); } list_free(indexes); return success; } /* * Close the partition identity indexes contained in the hash table and * destroy the hash table itself. */ static void close_partitions(partitions_hash *partitions) { partitions_iterator iterator; PartitionEntry *entry; partitions_start_iterate(partitions, &iterator); while ((entry = partitions_iterate(partitions, &iterator)) != NULL) { index_close(entry->ident_index, AccessShareLock); ExecDropSingleTupleTableSlot(entry->slot); ExecDropSingleTupleTableSlot(entry->slot_ind); if (entry->conv_map) free_conversion_map_ext(entry->conv_map); FreeBulkInsertState(entry->bistate); } partitions_destroy(partitions); } /* * Find hash entry for given partition. */ PartitionEntry * get_partition_entry(partitions_hash *partitions, Oid part_oid) { PartitionEntry *entry; entry = partitions_lookup(partitions, part_oid); if (entry == NULL) elog(ERROR, "bulk insert state not found for partition %u", part_oid); Assert(entry->part_oid == part_oid); return entry; } /* * Like make_attrmap() in PG core, but return AttrMapExt. */ static AttrMapExt * make_attrmap_ext(int maplen) { AttrMapExt *res; res = (AttrMapExt *) palloc0(sizeof(AttrMapExt)); res->maplen = maplen; res->attnums = (AttrNumber *) palloc0(sizeof(AttrNumber) * maplen); res->dropped_attr = false; res->exprsIn = palloc0_array(Node *, maplen); res->exprsOut = palloc0_array(Node *, maplen); return res; } static void free_attrmap_ext(AttrMapExt *map) { pfree(map->attnums); pfree(map->exprsIn); pfree(map->exprsOut); pfree(map); } /* * Like convert_tuples_by_name() in PG core, but try to coerce if the input * and output types differ. */ static TupleConversionMapExt * convert_tuples_by_name_ext(Relation rel_src, Relation rel_dst) { TupleDesc indesc = RelationGetDescr(rel_src); TupleDesc outdesc = RelationGetDescr(rel_dst); AttrMapExt *attrMap; /* Verify compatibility and prepare attribute-number map */ attrMap = build_attrmap_by_name_if_req_ext(rel_src, rel_dst); if (attrMap == NULL) { /* runtime conversion is not needed */ return NULL; } return convert_tuples_by_name_attrmap_ext(indesc, outdesc, attrMap); } /* * Like build_attrmap_by_name_if_req() in PG core, but try to coerce if the * input and output types differ. */ static AttrMapExt * build_attrmap_by_name_if_req_ext(Relation rel_src, Relation rel_dst) { TupleDesc indesc = RelationGetDescr(rel_src); TupleDesc outdesc = RelationGetDescr(rel_dst); AttrMapExt *attrMap; /* Verify compatibility and prepare attribute-number map */ attrMap = build_attrmap_by_name_ext(rel_src, rel_dst); /* * Check if the map has a one-to-one match and if there's any coercion. */ if (check_attrmap_match_ext(indesc, outdesc, attrMap)) { /* Runtime conversion is not needed */ free_attrmap_ext(attrMap); return NULL; } return attrMap; } /* * Like build_attrmap_by_name() in PG core but try to coerce if the input and * output types differ. */ static AttrMapExt * build_attrmap_by_name_ext(Relation rel_src, Relation rel_dst) { AttrMapExt *attrMap; int outnatts; int innatts; int i; int nextindesc = -1; ParseState *pstate = NULL; TupleDesc indesc = RelationGetDescr(rel_src); TupleDesc outdesc = RelationGetDescr(rel_dst); outnatts = outdesc->natts; innatts = indesc->natts; attrMap = make_attrmap_ext(outnatts); for (i = 0; i < outnatts; i++) { Form_pg_attribute outatt = TupleDescAttr(outdesc, i); char *attname; Oid atttypid; #if PG_VERSION_NUM >= 180000 Oid attcol; #endif int32 atttypmod; int j; if (outatt->attisdropped) { attrMap->dropped_attr = true; continue; /* attrMap->attnums[i] is already 0 */ } attname = NameStr(outatt->attname); atttypid = outatt->atttypid; atttypmod = outatt->atttypmod; #if PG_VERSION_NUM >= 180000 attcol = outatt->attcollation; #endif /* * Now search for an attribute with the same name in the indesc. It * seems likely that a partitioned table will have the attributes in * the same order as the partition, so the search below is optimized * for that case. It is possible that columns are dropped in one of * the relations, but not the other, so we use the 'nextindesc' * counter to track the starting point of the search. If the inner * loop encounters dropped columns then it will have to skip over * them, but it should leave 'nextindesc' at the correct position for * the next outer loop. */ for (j = 0; j < innatts; j++) { Form_pg_attribute inatt; nextindesc++; if (nextindesc >= innatts) nextindesc = 0; inatt = TupleDescAttr(indesc, nextindesc); if (inatt->attisdropped) { attrMap->dropped_attr = true; continue; } if (strcmp(attname, NameStr(inatt->attname)) == 0) { Node *expr; /* * Found it. Insert NULL into generated virtual columns, the * actual value will be computed during query execution. */ if (outatt->attgenerated == ATTRIBUTE_GENERATED_STORED) { /* * Initialize the expression to compute the stored value * of the column. * * This is redundant if the value in the input tuple * already has the correct value (typically because it's * generated by the same expression) but such conditions * are not trivial to check. */ expr = (Node *) build_generation_expression_ext(rel_dst, outatt->attnum); if (expr == NULL) /* This should not happen. */ ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("could not retrieve expression for attribute \"%s\" of relation \"%s\"", attname, RelationGetRelationName(rel_dst)))); /* * coerce_to_target_type() is not needed here - the * expression should have been coerced before it was * stored in the catalog. */ if (pstate == NULL) pstate = make_parsestate(NULL); assign_expr_collations(pstate, expr); attrMap->exprsOut[i] = expr; } #if PG_VERSION_NUM >= 180000 else if (outatt->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) attrMap->exprsOut[i] = (Node *) makeNullConst(atttypid, atttypmod, attcol); #endif /* * Check type. Also make sure that we have the expression to * generate the value of a virtual generated column. */ if (atttypid != inatt->atttypid || atttypmod != inatt->atttypmod #if PG_VERSION_NUM >= 180000 || inatt->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL #endif ) { /* * Can the input attribute be coerced to the output one? * * XXX Currently we follow ATPrepAlterColumnType() in PG * core - should anything be different? */ expr = (Node *) makeVar(1, inatt->attnum, inatt->atttypid, inatt->atttypmod, inatt->attcollation, 0); if (pstate == NULL) pstate = make_parsestate(NULL); expr = coerce_to_target_type(pstate, expr, exprType(expr), outatt->atttypid, outatt->atttypmod, COERCION_ASSIGNMENT, COERCE_IMPLICIT_CAST, -1); #if PG_VERSION_NUM >= 180000 /* Here we take the column expression into account. */ if (inatt->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) expr = expand_generated_columns_in_expr(expr, rel_src, 1); #endif /* * XXX Do we need to call expression_planner() like * ATPrepAlterColumnType() in PG core does? * ExecPrepareExpr() calls it before execution anyway. */ if (expr) { assign_expr_collations(pstate, expr); attrMap->exprsIn[i] = expr; } else ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("could not convert row type"), errdetail("Attribute \"%s\" of type %s does not match corresponding attribute of type %s.", attname, format_type_be(outdesc->tdtypeid), format_type_be(indesc->tdtypeid)))); } /* * XXX Probably not needed if we set attrMap->exprs...[i] * above. */ attrMap->attnums[i] = inatt->attnum; break; } } if (attrMap->attnums[i] == 0) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("could not convert row type"), errdetail("Attribute \"%s\" of type %s does not exist in type %s.", attname, format_type_be(outdesc->tdtypeid), format_type_be(indesc->tdtypeid)))); } return attrMap; } /* * check_attrmap_match() copied from PG core and adjusted so it takes coercion * into account. */ static bool check_attrmap_match_ext(TupleDesc indesc, TupleDesc outdesc, AttrMapExt *attrMap) { int i; /* * Dropped attribute in either descriptor makes the function return false, * even if it appears in both descriptors and at the same position. Thus * we (mis)use the map to get rid of the values of the dropped columns. */ if (attrMap->dropped_attr) return false; /* no match if attribute numbers are not the same */ if (indesc->natts != outdesc->natts) return false; /* no match if there is at least one expression. */ for (i = 0; i < attrMap->maplen; i++) { if (attrMap->exprsIn[i] || attrMap->exprsOut[i]) return false; } for (i = 0; i < attrMap->maplen; i++) { Form_pg_attribute inatt = TupleDescAttr(indesc, i); Form_pg_attribute outatt = TupleDescAttr(outdesc, i); /* * If the input column has a missing attribute, we need a conversion. */ if (inatt->atthasmissing) return false; if (attrMap->attnums[i] == (i + 1)) continue; /* * If it's a dropped column and the corresponding input column is also * dropped, we don't need a conversion. However, attlen and attalign * must agree. */ if (attrMap->attnums[i] == 0 && inatt->attisdropped && inatt->attlen == outatt->attlen && inatt->attalign == outatt->attalign) continue; return false; } return true; } /* * Like convert_tuples_by_name_attrmap() but handle coerce expressions. */ static TupleConversionMapExt * convert_tuples_by_name_attrmap_ext(TupleDesc indesc, TupleDesc outdesc, AttrMapExt *attrMap) { int n = outdesc->natts; TupleConversionMapExt *map; EState *estate; bool have_outer_expr = false; Assert(attrMap != NULL); /* Prepare the map structure */ map = (TupleConversionMapExt *) palloc0(sizeof(TupleConversionMapExt)); map->indesc = indesc; map->outdesc = outdesc; map->attrMap = attrMap; /* preallocate workspace for Datum arrays */ n = indesc->natts + 1; /* +1 for NULL */ map->invalues = (Datum *) palloc(n * sizeof(Datum)); map->inisnull = (bool *) palloc(n * sizeof(bool)); map->invalues[0] = (Datum) 0; /* set up the NULL entry */ map->inisnull[0] = true; map->exprsIn = (ExprState **) palloc0_array(ExprState *, indesc->natts); estate = CreateExecutorState(); for (int i = 0; i < outdesc->natts; i++) { Expr *expr = (Expr *) attrMap->exprsIn[i]; if (expr) map->exprsIn[i] = ExecPrepareExpr(expr, estate); if (attrMap->exprsOut[i]) have_outer_expr = true; } if (have_outer_expr) { Assert(indesc->natts == outdesc->natts); map->exprsOut = (ExprState **) palloc0_array(ExprState *, outdesc->natts); for (int i = 0; i < outdesc->natts; i++) { Expr *expr = (Expr *) attrMap->exprsOut[i]; if (expr) map->exprsOut[i] = ExecPrepareExpr(expr, estate); } } map->estate = estate; map->in_slot = MakeSingleTupleTableSlot(indesc, &TTSOpsHeapTuple); map->out_slot = MakeSingleTupleTableSlot(outdesc, &TTSOpsVirtual); return map; } /* * execute_attr_map_tuple() copied from PG core and adjusted to handle coerce * expressions. */ HeapTuple pg_rewrite_execute_attr_map_tuple(HeapTuple tuple, TupleConversionMapExt *map) { AttrMapExt *attrMap = map->attrMap; Datum *invalues = map->invalues; bool *inisnull = map->inisnull; Datum *outvalues = map->out_slot->tts_values; bool *outisnull = map->out_slot->tts_isnull; int i; ExprContext *ecxt; /* * Extract all the values of the old tuple, offsetting the arrays so that * invalues[0] is left NULL and invalues[1] is the first source attribute; * this exactly matches the numbering convention in attrMap. */ heap_deform_tuple(tuple, map->indesc, invalues + 1, inisnull + 1); /* Prepare for evaluation of expressions. */ ResetPerTupleExprContext(map->estate); ecxt = GetPerTupleExprContext(map->estate); ExecClearTuple(map->in_slot); ExecStoreHeapTuple(tuple, map->in_slot, false); ecxt->ecxt_scantuple = map->in_slot; if (map->out_slot) ExecClearTuple(map->out_slot); /* * Transpose into proper fields of the new tuple. */ Assert(attrMap->maplen == map->outdesc->natts); for (i = 0; i < attrMap->maplen; i++) { int j = attrMap->attnums[i]; ExprState *expr = map->exprsIn[i]; if (expr == NULL) { /* Simply copy the value. */ outvalues[i] = invalues[j]; outisnull[i] = inisnull[j]; } else { /* Generated column - evaluate the expression.. */ outvalues[i] = ExecEvalExprSwitchContext(expr, ecxt, &outisnull[i]); } } /* * Compute values of the ATTRIBUTE_GENERATED_STORED attributes in the * output tuple if there are some. */ if (map->exprsOut) { /* * The values are already in the slot's tts_values/tts_isnull arrays * because outvalues/outisnull are just pointers to these. */ ExecStoreVirtualTuple(map->out_slot); ecxt->ecxt_scantuple = map->out_slot; for (i = 0; i < attrMap->maplen; i++) { ExprState *expr = map->exprsOut[i]; /* * As noted in build_attrmap_by_name_ext(), the value may already * be correct if the same expression was used to generate (and * store) the it in the source table. However it's easier to * compute it again than to compare the expression. (Different * order of attributes can make the comparison tricky.) */ if (expr) outvalues[i] = ExecEvalExprSwitchContext(expr, ecxt, &outisnull[i]); } } /* * Form and return the new tuple. */ return heap_form_tuple(map->outdesc, outvalues, outisnull); } /* * free_conversion_map() copied from PG core and adjusted to handle coerce * expressions. */ static void free_conversion_map_ext(TupleConversionMapExt *map) { /* indesc and outdesc are not ours to free */ free_attrmap_ext(map->attrMap); pfree(map->invalues); pfree(map->inisnull); FreeExecutorState(map->estate); ExecDropSingleTupleTableSlot(map->in_slot); ExecDropSingleTupleTableSlot(map->out_slot); pfree(map); } #define TASK_LIST_RES_ATTRS 9 /* Get information on squeeze workers on the current database. */ PG_FUNCTION_INFO_V1(pg_rewrite_get_task_list); Datum pg_rewrite_get_task_list(PG_FUNCTION_ARGS) { WorkerTask *tasks, *dst; int i, ntasks = 0; #if PG_VERSION_NUM >= 150000 ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; InitMaterializedSRF(fcinfo, 0); #else FuncCallContext *funcctx; int call_cntr, max_calls; HeapTuple *tuples; #endif /* * Copy the task information at once. */ tasks = (WorkerTask *) palloc(MAX_TASKS * sizeof(WorkerTask)); dst = tasks; for (i = 0; i < MAX_TASKS; i++) { WorkerTask *task = &workerTasks[i]; Oid dbid; pid_t pid; SpinLockAcquire(&task->mutex); dbid = task->dbid; pid = task->pid; /* * A system call (see memcpy() below) while holding a spinlock is * probably not a good practice. */ SpinLockRelease(&task->mutex); if (dbid == MyDatabaseId && pid != InvalidPid) { memcpy(dst, task, sizeof(WorkerTask)); /* * Since we copied the data w/o locking, verify if the task is * still owned by the same backend and the same worker. (In * theory, PID could be reused by another worker by now, but it's * very unlikely and even if that happened, it cannot cause * anything like data corruption.) */ SpinLockAcquire(&task->mutex); if (task->dbid == dbid && task->pid == pid) { dst++; ntasks++; } SpinLockRelease(&task->mutex); } } #if PG_VERSION_NUM >= 150000 for (i = 0; i < ntasks; i++) { WorkerTask *task = &tasks[i]; TaskProgress *progress = &task->progress; Datum values[TASK_LIST_RES_ATTRS]; bool isnull[TASK_LIST_RES_ATTRS]; memset(isnull, false, TASK_LIST_RES_ATTRS * sizeof(bool)); if (strlen(NameStr(task->relschema)) > 0) values[0] = NameGetDatum(&task->relschema); else isnull[0] = true; values[1] = NameGetDatum(&task->relname); if (strlen(NameStr(task->relschema_dst)) > 0) values[2] = NameGetDatum(&task->relschema_dst); else isnull[2] = true; values[3] = NameGetDatum(&task->relname_dst); values[4] = NameGetDatum(&task->relname_new); values[5] = Int64GetDatum(progress->ins_initial); values[6] = Int64GetDatum(progress->ins); values[7] = Int64GetDatum(progress->upd); values[8] = Int64GetDatum(progress->del); tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, isnull); } return (Datum) 0; #else /* Less trivial implementation, to be removed when PG 14 is EOL. */ if (SRF_IS_FIRSTCALL()) { MemoryContext oldcontext; TupleDesc tupdesc; int ntuples = 0; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); /* XXX Is this necessary? */ funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc); /* Process only the slots that we really can display. */ tuples = (HeapTuple *) palloc0(ntasks * sizeof(HeapTuple)); for (i = 0; i < ntasks; i++) { WorkerTask *task = &tasks[i]; TaskProgress *progress = &task->progress; Datum *values; bool *isnull; values = (Datum *) palloc(TASK_LIST_RES_ATTRS * sizeof(Datum)); isnull = (bool *) palloc0(TASK_LIST_RES_ATTRS * sizeof(bool)); if (strlen(NameStr(task->relschema)) > 0) values[0] = NameGetDatum(&task->relschema); else isnull[0] = true; values[1] = NameGetDatum(&task->relname); if (strlen(NameStr(task->relschema_dst)) > 0) values[2] = NameGetDatum(&task->relschema_dst); else isnull[2] = true; values[3] = NameGetDatum(&task->relname_dst); values[4] = NameGetDatum(&task->relname_new); values[5] = Int64GetDatum(progress->ins_initial); values[6] = Int64GetDatum(progress->ins); values[7] = Int64GetDatum(progress->upd); values[8] = Int64GetDatum(progress->del); tuples[ntuples++] = heap_form_tuple(tupdesc, values, isnull); } funcctx->user_fctx = tuples; funcctx->max_calls = ntuples;; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); call_cntr = funcctx->call_cntr; max_calls = funcctx->max_calls; tuples = (HeapTuple *) funcctx->user_fctx; if (call_cntr < max_calls) { HeapTuple tuple = tuples[call_cntr]; Datum result; result = HeapTupleGetDatum(tuple); SRF_RETURN_NEXT(funcctx, result); } else SRF_RETURN_DONE(funcctx); #endif } /* * Create constraints on "destination relation" according to "source relation" * and mark them NOT VALID. * * Type conversion(s) that we've done during rewriting must not break any * constraints on the table. Even though all the tuples we insert (possibly * converted) into the destination tuple had to be validated in the source * table, we should be careful to say that the validity in the source table * implies validity in the destination table. An obvious example is that * float-to-int conversion on the FK side of an RI constraint can leave some * rows in FK table with no matching rows in the PK table. * * We don't have to address PK, UNIQUE and EXCLUDE constraints here because * these are enforced immediately as we run DMLs on the destination * table. Thus we only need to tell the user that he should create these * constraints. However, rewrite_table() works at low level and thus it * by-passes checking of the other kinds of constraints. * * One way to address this problem could be to create the FK, CHECK and NOT * NULL constraints on the rewritten table *after* the completion of * rewrite_table(), but that would leave the table w/o constraints for some * time. Moreover, AccessExclusiveLock would be needed for the constraint * creation. * * Another possible approach would be to create the constraints while we're * still holding AccessExclusiveLock (around the time we do table * renaming). That would never leave the table w/o constraints, but it would * still block access to the table for significant time. (Although it'd still * be better than regular ALTER TABLE ... ALTER COLUMN ... SET DATA TYPE * ... command, because this command holds the AccessExclusiveLock lock during * the actual rewriting.) * * The least disruptive approach is apparently that we create the FK and CHECK * constraints on the destination table and mark them NOT VALID, and let the * user validate them "manually". The validation only needs * ShareUpdateExclusiveLock, which does not block read / write access to the * table. (Note that all data changes performed after rewrite_table() has * finished are checked even with NOT VALID constraints.) * * Note on NOT NULL: this constraint cannot be created as NOT VALID in PG <= * 17, so the only perfect way to handle this one is to create it after the * completion of rewrite_table(). However, as type conversions usually do not * change non-NULL value to NULL, it's probably o.k. to create the constraint * before running rewrite_table(). * * We actually create the NOT VALID constraints even if there is no type * conversion - this is to avoid excessive blocking as explained * above. However, in this case we can then safely change the 'convalidated' * field of pg_constraint w/o actual validation. The user can also ask * rewrite_table() to fake the validation this way if he is confident that * particular data type conversion cannot affect validity of any * constraints. This is probably true in the (supposedly) most common case of * changing data type from integer to bigint. TODO The "fake validation" is * yet to be implemented. * * OID and name of the destination table is passed instead of an open relcache * entry because SPI requires the relation to be closed. We expect that the * both relations are locked using AccessExclusiveLock mode. */ static void copy_constraints(Oid relid_dst, const char *relname_dst, Oid relid_src) { Relation rel; TableScanDesc scan; HeapTuple tuple; StringInfo buf = makeStringInfo(); List *cmds = NIL; ListCell *lc; /* * Iterate through all the constraints as we need to check both conrelid * and confrelid (there's no index on the latter). */ rel = table_open(ConstraintRelationId, AccessShareLock); scan = table_beginscan_catalog(rel, 0, NULL); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_constraint con = (Form_pg_constraint) GETSTRUCT(tuple); resetStringInfo(buf); switch (con->contype) { case CONSTRAINT_CHECK: if (con->conrelid == relid_src) dump_check_constraint(relid_dst, relname_dst, tuple, buf); break; case CONSTRAINT_FOREIGN: { if (con->conrelid == relid_src || con->confrelid == relid_src) dump_fk_constraint(tuple, relid_dst, relname_dst, relid_src, buf); break; } #if PG_VERSION_NUM >= 180000 case CONSTRAINT_NOTNULL: { if (con->conrelid == relid_src) dump_null_constraint(relid_dst, relname_dst, tuple, buf); break; } #endif default: break; } /* Add the DDL to a list. */ if (strlen(buf->data) > 0) { appendStringInfoString(buf, " NOT VALID"); cmds = lappend(cmds, pstrdup(buf->data)); } } table_endscan(scan); table_close(rel, AccessShareLock); if (cmds == NIL) return; /* Run the commands. */ SPI_connect(); PushActiveSnapshot(GetTransactionSnapshot()); foreach(lc, cmds) { char *cmd = (char *) lfirst(lc); int ret; ret = SPI_execute(cmd, false, 0); if (ret != SPI_OK_UTILITY) ereport(ERROR, (errmsg("command failed: \"%s\"", cmd))); } PopActiveSnapshot(); SPI_finish(); list_free_deep(cmds); } static void dump_fk_constraint(HeapTuple tup, Oid relid_dst, const char *relname_dst, Oid relid_src, StringInfo buf) { Oid relid_other; const char *pkrelname, *pknsp, *fkrelname, *fknsp; Form_pg_constraint con = (Form_pg_constraint) GETSTRUCT(tup); Datum val; #if PG_VERSION_NUM >= 150000 bool isnull; #endif const char *string; Assert(con->contype == CONSTRAINT_FOREIGN); if (con->conrelid == relid_src) { #if PG_VERSION_NUM < 180000 /* * ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY ... NOT VALID is not * supported for partitioned FK table in PG < 18. */ if (get_rel_relkind(relid_dst) == RELKIND_PARTITIONED_TABLE) { send_message(MyWorkerTask, NOTICE, "FOREIGN KEY with NOT VALID option cannot be added to partitioned table", NULL); return; } #endif fknsp = get_namespace_name(relid_dst); fkrelname = relname_dst; relid_other = con->confrelid; pknsp = get_namespace_name(relid_other); pkrelname = get_rel_name(relid_other); } else { Assert(con->confrelid == relid_src); /* * Like above, but check the existing FK table (because what we * rewrite now is the PK table.) */ if (get_rel_relkind(con->conrelid) == RELKIND_PARTITIONED_TABLE) return; pknsp = get_namespace_name(relid_dst); pkrelname = relname_dst; relid_other = con->conrelid; fknsp = get_namespace_name(relid_other); fkrelname = get_rel_name(relid_other); } dump_constraint_common(fknsp, fkrelname, con, buf); /* * The rest is mostly copied from pg_get_constraintdef_worker() in PG * core. */ /* Start off the constraint definition */ appendStringInfoString(buf, "FOREIGN KEY ("); /* Fetch and build referencing-column list */ #if PG_VERSION_NUM >= 160000 val = SysCacheGetAttrNotNull(CONSTROID, tup, Anum_pg_constraint_conkey); #else { bool isnull; val = SysCacheGetAttr(CONSTROID, tup, Anum_pg_constraint_conkey, &isnull); Assert(!isnull); } #endif decompile_column_index_array(val, con->conrelid, buf); /* add foreign relation name */ appendStringInfo(buf, ") REFERENCES %s(", quote_qualified_identifier(pknsp, pkrelname)); /* Fetch and build referenced-column list */ #if PG_VERSION_NUM >= 160000 val = SysCacheGetAttrNotNull(CONSTROID, tup, Anum_pg_constraint_confkey); #else { bool isnull; val = SysCacheGetAttr(CONSTROID, tup, Anum_pg_constraint_confkey, &isnull); Assert(!isnull); } #endif decompile_column_index_array(val, con->confrelid, buf); appendStringInfoChar(buf, ')'); /* Add match type */ switch (con->confmatchtype) { case FKCONSTR_MATCH_FULL: string = " MATCH FULL"; break; case FKCONSTR_MATCH_PARTIAL: string = " MATCH PARTIAL"; break; case FKCONSTR_MATCH_SIMPLE: string = ""; break; default: elog(ERROR, "unrecognized confmatchtype: %d", con->confmatchtype); string = ""; /* keep compiler quiet */ break; } appendStringInfoString(buf, string); /* Add ON UPDATE and ON DELETE clauses, if needed */ switch (con->confupdtype) { case FKCONSTR_ACTION_NOACTION: string = NULL; /* suppress default */ break; case FKCONSTR_ACTION_RESTRICT: string = "RESTRICT"; break; case FKCONSTR_ACTION_CASCADE: string = "CASCADE"; break; case FKCONSTR_ACTION_SETNULL: string = "SET NULL"; break; case FKCONSTR_ACTION_SETDEFAULT: string = "SET DEFAULT"; break; default: elog(ERROR, "unrecognized confupdtype: %d", con->confupdtype); string = NULL; /* keep compiler quiet */ break; } if (string) appendStringInfo(buf, " ON UPDATE %s", string); switch (con->confdeltype) { case FKCONSTR_ACTION_NOACTION: string = NULL; /* suppress default */ break; case FKCONSTR_ACTION_RESTRICT: string = "RESTRICT"; break; case FKCONSTR_ACTION_CASCADE: string = "CASCADE"; break; case FKCONSTR_ACTION_SETNULL: string = "SET NULL"; break; case FKCONSTR_ACTION_SETDEFAULT: string = "SET DEFAULT"; break; default: elog(ERROR, "unrecognized confdeltype: %d", con->confdeltype); string = NULL; /* keep compiler quiet */ break; } if (string) appendStringInfo(buf, " ON DELETE %s", string); #if PG_VERSION_NUM >= 150000 /* * Add columns specified to SET NULL or SET DEFAULT if * provided. */ val = SysCacheGetAttr(CONSTROID, tup, Anum_pg_constraint_confdelsetcols, &isnull); if (!isnull) { appendStringInfoString(buf, " ("); decompile_column_index_array(val, con->conrelid, buf); appendStringInfoChar(buf, ')'); } #endif } /* * Mostly copied from pg_get_constraintdef_worker() in PG core. */ static void dump_check_constraint(Oid relid_dst, const char *relname_dst, HeapTuple tup, StringInfo buf) { Datum val; char *conbin; char *consrc; Node *expr; List *context; Form_pg_constraint con; const char *nsp; con = (Form_pg_constraint) GETSTRUCT(tup); nsp = get_namespace_name(relid_dst); dump_constraint_common(nsp, relname_dst, con, buf); /* Fetch constraint expression in parsetree form */ #if PG_VERSION_NUM >= 160000 val = SysCacheGetAttrNotNull(CONSTROID, tup, Anum_pg_constraint_conbin); #else { bool isnull; val = SysCacheGetAttr(CONSTROID, tup, Anum_pg_constraint_conbin, &isnull); Assert(!isnull); } #endif conbin = TextDatumGetCString(val); expr = stringToNode(conbin); /* Set up deparsing context for Var nodes in constraint */ if (con->conrelid != InvalidOid) { /* relation constraint */ context = deparse_context_for(get_rel_name(con->conrelid), con->conrelid); } else { /* domain constraint --- can't have Vars */ context = NIL; } consrc = deparse_expression(expr, context, false, false); /* * Now emit the constraint definition, adding NO INHERIT if * necessary. * * There are cases where the constraint expression will be * fully parenthesized and we don't need the outer parens ... * but there are other cases where we do need 'em. Be * conservative for now. * * Note that simply checking for leading '(' and trailing ')' * would NOT be good enough, consider "(x > 0) AND (y > 0)". */ appendStringInfo(buf, "CHECK (%s)%s", consrc, con->connoinherit ? " NO INHERIT" : ""); } #if PG_VERSION_NUM >= 180000 static void dump_null_constraint(Oid relid_dst, const char *relname_dst, HeapTuple tup, StringInfo buf) { Form_pg_constraint con = (Form_pg_constraint) GETSTRUCT(tup); const char *nsp; AttrNumber attnum; Assert(con->contype == CONSTRAINT_NOTNULL); nsp = get_namespace_name(relid_dst); dump_constraint_common(nsp, relname_dst, con, buf); attnum = extractNotNullColumn(tup); appendStringInfo(buf, "NOT NULL %s", quote_identifier(get_attname(con->conrelid, attnum, false))); if (((Form_pg_constraint) GETSTRUCT(tup))->connoinherit) appendStringInfoString(buf, " NO INHERIT"); } #endif static void dump_constraint_common(const char *nsp, const char *relname, Form_pg_constraint con, StringInfo buf) { NameData conname_new; int conname_len = strlen(NameStr(con->conname)); if ((conname_len + 1) == NAMEDATALEN) /* * XXX Is it worth generating an unique name in another way? Not sure, * smart user can rename the original constraint. */ ereport(ERROR, (errmsg("constraint name \"%s\" is too long, cannot add suffix", NameStr(con->conname)))); else { namestrcpy(&conname_new, NameStr(con->conname)); /* Add '2' as a suffix. */ NameStr(conname_new)[conname_len] = '2'; } appendStringInfo(buf, "ALTER TABLE %s ADD CONSTRAINT %s ", quote_qualified_identifier(nsp, relname), quote_identifier(NameStr(conname_new))); } /* * Copied from PG core. */ static int decompile_column_index_array(Datum column_index_array, Oid relId, StringInfo buf) { Datum *keys; int nKeys; int j; /* Extract data from array of int16 */ #if PG_VERSION_NUM >= 160000 deconstruct_array_builtin(DatumGetArrayTypeP(column_index_array), INT2OID, &keys, NULL, &nKeys); #else deconstruct_array(DatumGetArrayTypeP(column_index_array), INT2OID, sizeof(int16), true, TYPALIGN_SHORT, &keys, NULL, &nKeys); #endif for (j = 0; j < nKeys; j++) { char *colName; colName = get_attname(relId, DatumGetInt16(keys[j]), false); if (j == 0) appendStringInfoString(buf, quote_identifier(colName)); else appendStringInfo(buf, ", %s", quote_identifier(colName)); } return nKeys; } /* * Like build_generation_expression() in PG core but no assertions about * virtual columns. */ static Node * build_generation_expression_ext(Relation rel, int attrno) { TupleDesc rd_att = RelationGetDescr(rel); Form_pg_attribute att_tup = TupleDescAttr(rd_att, attrno - 1); Node *defexpr; Oid attcollid; /* * Assert(rd_att->constr && rd_att->constr->has_generated_virtual); * Assert(att_tup->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL); */ defexpr = build_column_default(rel, attrno); if (defexpr == NULL) elog(ERROR, "no generation expression found for column number %d of table \"%s\"", attrno, RelationGetRelationName(rel)); /* * If the column definition has a collation and it is different from the * collation of the generation expression, put a COLLATE clause around the * expression. */ attcollid = att_tup->attcollation; if (attcollid && attcollid != exprCollation(defexpr)) { CollateExpr *ce = makeNode(CollateExpr); ce->arg = (Expr *) defexpr; ce->collOid = attcollid; ce->location = -1; defexpr = (Node *) ce; } return defexpr; } pg_rewrite-REL2_0_0/pg_rewrite.control000066400000000000000000000002511504513220300201270ustar00rootroot00000000000000# pg_rewrite extension comment = 'Tool for maintenance that requires table rewriting.' default_version = '2.0' module_pathname = '$libdir/pg_rewrite' relocatable = true pg_rewrite-REL2_0_0/pg_rewrite.h000066400000000000000000000174671504513220300167170ustar00rootroot00000000000000/*---------------------------------------------------------------- * * pg_rewrite.h * Tools for maintenance that requires table rewriting. * * Copyright (c) 2021-2025, Cybertec PostgreSQL International GmbH * *---------------------------------------------------------------- */ #include #include "c.h" #include "postgres.h" #include "fmgr.h" #include "miscadmin.h" #include "access/genam.h" #include "access/heapam.h" #include "access/relscan.h" #include "access/xlog_internal.h" #include "access/xact.h" #include "catalog/pg_class.h" #include "nodes/execnodes.h" #include "postmaster/bgworker.h" #include "replication/logical.h" #include "replication/origin.h" #include "utils/inval.h" #include "utils/resowner.h" #include "utils/snapmgr.h" typedef struct DecodingOutputState { /* The relation whose changes we're decoding. */ Oid relid; /* * Decoded changes are stored here. Although we try to avoid excessive * batches, it can happen that the changes need to be stored to disk. The * tuplestore does this transparently. */ Tuplestorestate *tstore; /* The current number of changes in tstore. */ double nchanges; /* * Descriptor to store the ConcurrentChange structure serialized (bytea). * We can't store the tuple directly because tuplestore only supports * minimum tuple and we may need to transfer OID system column from the * output plugin. Also we need to transfer the change kind, so it's better * to put everything in the structure than to use 2 tuplestores "in * parallel". */ TupleDesc tupdesc_change; /* * Tuple descriptor needed process the concurrent data changes. */ TupleDesc tupdesc_src; /* Slot to retrieve data from tstore. */ TupleTableSlot *tsslot; /* * WAL records having this origin have been created by the initial load * and should not be decoded. */ RepOriginId rorigin; ResourceOwner resowner; } DecodingOutputState; /* The WAL segment being decoded. */ extern XLogSegNo rewrite_current_segment; extern void _PG_init(void); /* Progress tracking. */ typedef struct TaskProgress { /* Tuples inserted during the initial load. */ int64 ins_initial; /* * Tuples inserted, updated and deleted after the initial load (i.e. * during the catch-up phase). */ int64 ins; int64 upd; int64 del; } TaskProgress; /* * The new implementation, which delegates the execution to a background * worker (as opposed to the PG executor). * * Arguments are passed to the worker via this structure, located in the * shared memory. */ typedef struct WorkerTask { /* Connection info. */ Oid dbid; Oid roleid; /* Worker that performs the task both sets and clears this field. */ pid_t pid; /* See the comments of pg_rewrite_exit_if_requested(). */ bool exit_requested; /* The progress is only valid if the dbid is valid. */ TaskProgress progress; /* * Use this when setting / clearing the fields above. Once dbid is set, * the task belongs to the backend that set it, so the other fields may be * assigned w/o the lock. */ slock_t mutex; /* The tables to work on. */ NameData relschema; NameData relname; NameData relname_new; NameData relschema_dst; NameData relname_dst; /* * Space for the worker to send an error message to the backend. * * XXX Note that later messages overwrite the earlier ones, so only the * last message is received. Is it worth using a queue instead? */ #define MAX_ERR_MSG_LEN 1024 char msg[MAX_ERR_MSG_LEN]; /* Detailed error message. */ char msg_detail[MAX_ERR_MSG_LEN]; int elevel; /* * Should rewrite_table() return w/o waiting for the worker's exit? If * this flag is set, the worker is responsible for releasing the * task. Otherwise the worker must not release the task because the * backend might be interested in 'msg' and 'msg_detail'. */ bool nowait; int max_xlock_time; } WorkerTask; #define MAX_TASKS 8 /* Each backend stores here the pointer to its task in the shared memory. */ extern WorkerTask *MyWorkerTask; /* * Like AttrMap in PG core, but here we add an array of expressions to coerce * the input values to output ones. (A new name is needed as it's hard to * avoid inclusion of the in-core structure.) */ typedef struct AttrMapExt { AttrNumber *attnums; int maplen; bool dropped_attr; /* Has outer or inner descriptor a dropped * attribute? */ Node **exprsIn; /* Non-NULL field tells how to convert the input * value to the output data type and/or to * evaluate the column expression. NULL indicates * that no conversion is needed and that there is * no expression for given column. */ Node **exprsOut; /* * Likewise, expression to compute the value of an * output column. */ } AttrMapExt; /* * Like TupleConversionMap in PG core, but here we add an array of expressions * to coerce the input values to output ones. (A new name is needed as it's * hard to avoid inclusion of the in-core structure.) */ typedef struct TupleConversionMapExt { TupleDesc indesc; /* tupdesc for source rowtype */ TupleDesc outdesc; /* tupdesc for result rowtype */ AttrMapExt *attrMap; /* indexes of input fields, or 0 for null */ Datum *invalues; /* workspace for deconstructing source */ bool *inisnull; ExprState **exprsIn; /* See AttrMapExt */ ExprState **exprsOut; /* See AttrMapExt */ EState *estate; /* Executor state used to evaluate * coerceExprs. */ TupleTableSlot *in_slot; /* Slot to store the input tuple for * coercion. */ TupleTableSlot *out_slot; /* Slot to construct the output tuple. */ } TupleConversionMapExt; /* * Hash table to cache partition-specific information. */ typedef struct PartitionEntry { Oid part_oid; /* key */ Relation ident_index; /* * Slot (TTSOpsHeapTuple) to apply data changes to the partition. */ TupleTableSlot *slot; /* * Slot to retrieve tuples from the partition. Separate from 'slot_ind' * because it has to be TTSOpsBufferHeapTuple. */ TupleTableSlot *slot_ind; /* This should make insertions into partitions more efficient. */ BulkInsertState bistate; /* * Map to convert tuples that match the partitioned table so they match * this partition. */ TupleConversionMapExt *conv_map; char status; /* used by simplehash */ } PartitionEntry; #define SH_PREFIX partitions #define SH_ELEMENT_TYPE PartitionEntry #define SH_KEY_TYPE Oid #define SH_KEY part_oid #define SH_HASH_KEY(tb, key) (key) #define SH_EQUAL(tb, a, b) ((a) == (b)) #define SH_SCOPE static inline #define SH_DECLARE #define SH_DEFINE #include "lib/simplehash.h" extern PGDLLEXPORT void rewrite_worker_main(Datum main_arg); extern void pg_rewrite_exit_if_requested(void); /* * Use function names distinct from those in pg_squeeze, in case both * extensions are installed. */ extern bool pg_rewrite_process_concurrent_changes(EState *estate, ModifyTableState *mtstate, struct PartitionTupleRouting *proute, LogicalDecodingContext *ctx, XLogRecPtr end_of_wal, ScanKey ident_key, int ident_key_nentries, Relation ident_index, TupleTableSlot *slot_dst_ind, LOCKMODE lock_held, partitions_hash *partitions, TupleConversionMapExt *conv_map, struct timeval *must_complete); extern bool pg_rewrite_decode_concurrent_changes(LogicalDecodingContext *ctx, XLogRecPtr end_of_wal, struct timeval *must_complete); extern HeapTuple convert_tuple_for_dest_table(HeapTuple tuple, TupleConversionMapExt *conv_map); extern void _PG_output_plugin_init(OutputPluginCallbacks *cb); extern PartitionEntry *get_partition_entry(partitions_hash *partitions, Oid part_oid);; extern HeapTuple pg_rewrite_execute_attr_map_tuple(HeapTuple tuple, TupleConversionMapExt *map); pg_rewrite-REL2_0_0/pg_rewrite.md000077700000000000000000000000001504513220300203232README.mdustar00rootroot00000000000000pg_rewrite-REL2_0_0/specs/000077500000000000000000000000001504513220300154755ustar00rootroot00000000000000pg_rewrite-REL2_0_0/specs/pg_rewrite_concurrent.spec000066400000000000000000000062571504513220300227740ustar00rootroot00000000000000setup { CREATE EXTENSION injection_points; CREATE EXTENSION pg_rewrite; CREATE TABLE tbl_src(i int primary key, j int, k int generated always as (-j) virtual, l int generated always as (-j) stored); INSERT INTO tbl_src(i, j) VALUES (1, 10), (4, 40), (7, 70); -- Change of data type and column order. CREATE TABLE tbl_dst(j int, i bigint primary key, k int, l int); } teardown { DROP EXTENSION injection_points; DROP EXTENSION pg_rewrite; DROP TABLE tbl_src; DROP TABLE tbl_src_old; } session s1 setup { SELECT injection_points_attach('pg_rewrite-before-lock', 'wait'); SELECT injection_points_attach('pg_rewrite-after-commit', 'wait'); } # Perform the initial load and wait for s2 to do some data changes. # # Since pg_rewrite uses background worker, the isolation tester does not # recognize that the session waits on an injection point (because the worker # is who waits). Therefore use rewrite_table_nowait(), which only launches the # worker and goes on. The 'wait_for_s1_sleep' step below then checks until the # waiting started. step do_rewrite { SELECT rewrite_table_nowait('tbl_src', 'tbl_dst', 'tbl_src_old'); } # Check the data. step do_check { TABLE pg_rewrite_progress; SELECT i, j, k, l FROM tbl_src ORDER BY i, j; } session s2 # Since s1 uses background worker, the backend executing 'wait_before_lock' # does not appear to be waiting on the injection point. Instead we need to # check explicitly if the waiting on the injection point is in progress, and # wait if it's not. step wait_for_before_lock_ip { DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-before-lock'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; } step do_changes { INSERT INTO tbl_src VALUES (2, 20), (3, 30), (5, 50); -- Update with no identity change. UPDATE tbl_src SET j=0 WHERE i=1; -- Update with identity change. UPDATE tbl_src SET i=6 WHERE i=4; -- Update a row we inserted, to check that the insertion is visible. UPDATE tbl_src SET j=7 WHERE i=2; -- ... and update it again, to check that the update is visible. UPDATE tbl_src SET j=8 WHERE j=7; -- Delete. DELETE FROM tbl_src WHERE i=7; } step wakeup_before_lock_ip { SELECT injection_points_wakeup('pg_rewrite-before-lock'); } # Wait until the concurrent changes have been committed by the pg_rewrite # worker. step wait_for_after_commit_ip { DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-after-commit'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; } # Like wakeup_before_lock_ip above. step wakeup_after_commit_ip { SELECT injection_points_wakeup('pg_rewrite-after-commit'); } teardown { SELECT injection_points_detach('pg_rewrite-before-lock'); SELECT injection_points_detach('pg_rewrite-after-commit'); } permutation do_rewrite wait_for_before_lock_ip do_changes wakeup_before_lock_ip wait_for_after_commit_ip do_check wakeup_after_commit_ip pg_rewrite-REL2_0_0/specs/pg_rewrite_concurrent_partition.spec000066400000000000000000000067321504513220300250630ustar00rootroot00000000000000setup { CREATE EXTENSION injection_points; CREATE EXTENSION pg_rewrite; CREATE TABLE tbl_src(i int primary key, j int); INSERT INTO tbl_src(i, j) VALUES (1, 10), (4, 40); -- Besides partitioning, also test change of column type (int -> bigint). CREATE TABLE tbl_dst(i bigint primary key, j int) PARTITION BY RANGE(i); CREATE TABLE tbl_dst_part_1 PARTITION OF tbl_dst FOR VALUES FROM (1) TO (4); -- Create a partition with different order of columns, to test that -- partition maps work. CREATE TABLE tbl_dst_part_2(j int, i bigint primary key); ALTER TABLE tbl_dst ATTACH PARTITION tbl_dst_part_2 FOR VALUES FROM (4) TO (8); } teardown { DROP EXTENSION injection_points; DROP EXTENSION pg_rewrite; DROP TABLE tbl_src; DROP TABLE tbl_src_old; } session s1 setup { SELECT injection_points_attach('pg_rewrite-before-lock', 'wait'); SELECT injection_points_attach('pg_rewrite-after-commit', 'wait'); } # Perform the initial load and wait for s2 to do some data changes. # # Since pg_rewrite uses background worker, the isolation tester does not # recognize that the session waits on an injection point (because the worker # is who waits). Therefore use rewrite_table_nowait(), which only launches the # worker and goes on. The 'wait_for_s1_sleep' step below then checks until the # waiting started. step do_rewrite { SELECT rewrite_table_nowait('tbl_src', 'tbl_dst', 'tbl_src_old'); } # Check the data. step do_check { TABLE pg_rewrite_progress; SELECT i, j FROM tbl_src ORDER BY i, j; } session s2 # Since s1 uses background worker, the backend executing 'wait_before_lock' # does not appear to be waiting on the injection point. Instead we need to # check explicitly if the waiting on the injection point is in progress, and # wait if it's not. step wait_for_before_lock_ip { DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-before-lock'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; } step do_changes { -- Insert one row into each partition. INSERT INTO tbl_src VALUES (2, 20), (3, 30), (5, 50); -- Update with no identity change. UPDATE tbl_src SET j=0 WHERE i=1; -- Update with identity change but within the same partition. UPDATE tbl_src SET i=6 WHERE i=5; -- Cross-partition update. UPDATE tbl_src SET i=7 WHERE i=3; -- Update a row we inserted and updated, to check that it's visible. UPDATE tbl_src SET j=4 WHERE i=7; -- Delete. DELETE FROM tbl_src WHERE i=4; } step wakeup_before_lock_ip { SELECT injection_points_wakeup('pg_rewrite-before-lock'); } # Wait until the concurrent changes have been committed by the pg_rewrite # worker. step wait_for_after_commit_ip { DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-after-commit'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$ } # Like wakeup_before_lock_ip above. step wakeup_after_commit_ip { SELECT injection_points_wakeup('pg_rewrite-after-commit'); } teardown { SELECT injection_points_detach('pg_rewrite-before-lock'); SELECT injection_points_detach('pg_rewrite-after-commit'); } permutation do_rewrite wait_for_before_lock_ip do_changes wakeup_before_lock_ip wait_for_after_commit_ip do_check wakeup_after_commit_ip pg_rewrite-REL2_0_0/specs/pg_rewrite_concurrent_toast.spec000066400000000000000000000061411504513220300241760ustar00rootroot00000000000000setup { CREATE EXTENSION injection_points; CREATE EXTENSION pg_rewrite; CREATE TABLE tbl_src(i int primary key, t text); INSERT INTO tbl_src(i, t) SELECT x, string_agg(random()::text, '') FROM generate_series(1, 2) g(x), generate_series(1, 200) h(y) GROUP BY x; CREATE TABLE tbl_dst(i int primary key, t text); } teardown { DROP EXTENSION injection_points; DROP EXTENSION pg_rewrite; DROP TABLE tbl_src; DROP TABLE tbl_src_old; } session s1 setup { SELECT injection_points_attach('pg_rewrite-before-lock', 'wait'); SELECT injection_points_attach('pg_rewrite-after-commit', 'wait'); } # Perform the initial load and wait for s2 to do some data changes. # # Since pg_rewrite uses background worker, the isolation tester does not # recognize that the session waits on an injection point (because the worker # is who waits). Therefore use rewrite_table_nowait(), which only launches the # worker and goes on. The 'wait_for_s1_sleep' step below then checks until the # waiting started. step do_rewrite { SELECT rewrite_table_nowait('tbl_src', 'tbl_dst', 'tbl_src_old'); } # Check the data. step do_check { TABLE pg_rewrite_progress; -- Each row should contain TOASTed value. SELECT count(*) FROM tbl_src WHERE pg_column_toast_chunk_id(t) ISNULL; -- The contents of the new table should be identical to that of the old -- one. SELECT count(*) FROM tbl_src t1 JOIN tbl_src_old t2 ON t1.i = t2.i WHERE t1.t <> t2.t; } session s2 # Since s1 uses background worker, the backend executing 'wait_before_lock' # does not appear to be waiting on the injection point. Instead we need to # check explicitly if the waiting on the injection point is in progress, and # wait if it's not. step wait_for_before_lock_ip { DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-before-lock'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; } step do_changes { INSERT INTO tbl_src(i, t) SELECT 5, string_agg(random()::text, '') FROM generate_series(1, 200) h(y); UPDATE tbl_src SET t = t || 'x' WHERE i = 1; } step wakeup_before_lock_ip { SELECT injection_points_wakeup('pg_rewrite-before-lock'); } # Wait until the concurrent changes have been committed by the pg_rewrite # worker. step wait_for_after_commit_ip { DO $$ BEGIN LOOP PERFORM pg_stat_clear_snapshot(); PERFORM FROM pg_stat_activity WHERE (wait_event_type, wait_event)=('InjectionPoint', 'pg_rewrite-after-commit'); IF FOUND THEN EXIT; END IF; PERFORM pg_sleep(.1); END LOOP; END; $$; } # Like wakeup_before_lock_ip above. step wakeup_after_commit_ip { SELECT injection_points_wakeup('pg_rewrite-after-commit'); } teardown { SELECT injection_points_detach('pg_rewrite-before-lock'); SELECT injection_points_detach('pg_rewrite-after-commit'); } permutation do_rewrite wait_for_before_lock_ip do_changes wakeup_before_lock_ip wait_for_after_commit_ip do_check wakeup_after_commit_ip pg_rewrite-REL2_0_0/sql/000077500000000000000000000000001504513220300151575ustar00rootroot00000000000000pg_rewrite-REL2_0_0/sql/generated.sql000066400000000000000000000030351504513220300176370ustar00rootroot00000000000000-- Generated columns - some meaningful combinations of source and destination -- columns. CREATE TABLE tab7( i int primary key, j int, k int generated always as (i + 1) virtual, l int generated always AS (i + 1) stored, m int generated always AS (i + 1) virtual); CREATE TABLE tab7_new( i int primary key, -- Override the value copied from the source table. j int generated always AS (i - 1) stored, -- Check that the expression is evaluated correctly on the source -- table. k int, -- The same for stored expression. l int, -- Override the value computed on the source table. m int generated always as (i - 1) virtual); INSERT INTO tab7(i, j) VALUES (1, 1); SELECT rewrite_table('tab7', 'tab7_new', 'tab7_orig'); SELECT * FROM tab7; CREATE EXTENSION pageinspect; -- HEAP_HASNULL indicates that the value of 'm' hasn't been copied from the -- source table. SELECT raw_flags FROM heap_page_items(get_raw_page('tab7', 0)), LATERAL heap_tuple_infomask_flags(t_infomask, t_infomask2); -- For PG < 18, test without VIRTUAL columns. CREATE TABLE tab8( i int primary key, j int, k int generated always AS (i + 1) stored); CREATE TABLE tab8_new( i int primary key, -- Override the value copied from the source table. j int generated always AS (i - 1) stored, -- Check that the expression is evaluated correctly on the source -- table. k int); INSERT INTO tab8(i, j) VALUES (1, 1); SELECT rewrite_table('tab8', 'tab8_new', 'tab8_orig'); SELECT * FROM tab8; pg_rewrite-REL2_0_0/sql/pg_rewrite.sql000066400000000000000000000125571504513220300200610ustar00rootroot00000000000000DROP EXTENSION IF EXISTS pg_rewrite; CREATE EXTENSION pg_rewrite; CREATE TABLE tab1(i int PRIMARY KEY, j int, k int); -- If a dropped column is encountered, the source tuple should be converted -- so it matches the destination table. ALTER TABLE tab1 DROP COLUMN k; ALTER TABLE tab1 ADD COLUMN k int; INSERT INTO tab1(i, j, k) SELECT i, i / 2, i FROM generate_series(0, 1023) g(i); CREATE TABLE tab1_new(i int PRIMARY KEY, j int, k int) PARTITION BY RANGE(i); CREATE TABLE tab1_new_part_1 PARTITION OF tab1_new FOR VALUES FROM (0) TO (256); CREATE TABLE tab1_new_part_2 PARTITION OF tab1_new FOR VALUES FROM (256) TO (512); CREATE TABLE tab1_new_part_3 PARTITION OF tab1_new FOR VALUES FROM (512) TO (768); CREATE TABLE tab1_new_part_4 PARTITION OF tab1_new FOR VALUES FROM (768) TO (1024); -- Also test handling of constraints that require "manual" validation. ALTER TABLE tab1 ADD CHECK (k >= 0); CREATE TABLE tab1_fk(i int REFERENCES tab1); INSERT INTO tab1_fk(i) VALUES (1); \d tab1 -- Process the table. SELECT rewrite_table('tab1', 'tab1_new', 'tab1_orig'); -- tab1 should now be partitioned. \d tab1 -- Validate the constraints. ALTER TABLE tab1 VALIDATE CONSTRAINT tab1_k_check2; ALTER TABLE tab1_fk VALIDATE CONSTRAINT tab1_fk_i_fkey2; \d tab1 EXPLAIN (COSTS off) SELECT * FROM tab1; -- Check that the contents has not changed. SELECT count(*) FROM tab1; SELECT * FROM tab1 t FULL JOIN tab1_orig o ON t.i = o.i WHERE t.i ISNULL OR o.i ISNULL; -- List partitioning CREATE TABLE tab2(i int, j int, PRIMARY KEY (i, j)); INSERT INTO tab2(i, j) SELECT i, j FROM generate_series(1, 4) g(i), generate_series(1, 4) h(j); CREATE TABLE tab2_new(i int, j int, PRIMARY KEY (i, j)) PARTITION BY LIST(i); CREATE TABLE tab2_new_part_1 PARTITION OF tab2_new FOR VALUES IN (1); CREATE TABLE tab2_new_part_2 PARTITION OF tab2_new FOR VALUES IN (2); CREATE TABLE tab2_new_part_3 PARTITION OF tab2_new FOR VALUES IN (3); CREATE TABLE tab2_new_part_4 PARTITION OF tab2_new FOR VALUES IN (4); SELECT rewrite_table('tab2', 'tab2_new', 'tab2_orig'); TABLE tab2_new_part_1; TABLE tab2_new_part_2; TABLE tab2_new_part_3; TABLE tab2_new_part_4; -- Hash partitioning CREATE TABLE tab3(i int, j int, PRIMARY KEY (i, j)); INSERT INTO tab3(i, j) SELECT i, j FROM generate_series(1, 4) g(i), generate_series(1, 4) h(j); CREATE TABLE tab3_new(i int, j int, PRIMARY KEY (i, j)) PARTITION BY HASH(i); CREATE TABLE tab3_new_part_1 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 0); CREATE TABLE tab3_new_part_2 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 1); CREATE TABLE tab3_new_part_3 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 2); CREATE TABLE tab3_new_part_4 PARTITION OF tab3_new FOR VALUES WITH (MODULUS 4, REMAINDER 3); SELECT rewrite_table('tab3', 'tab3_new', 'tab3_orig'); TABLE tab3_new_part_1; TABLE tab3_new_part_2; TABLE tab3_new_part_3; TABLE tab3_new_part_4; -- Change of precision and scale of a numeric data type. CREATE TABLE tab4(i int PRIMARY KEY, j numeric(3, 1)); INSERT INTO tab4(i, j) VALUES (1, 0.1); CREATE TABLE tab4_new(i int PRIMARY KEY, j numeric(4, 2)); TABLE tab4; SELECT rewrite_table('tab4', 'tab4_new', 'tab4_orig'); TABLE tab4; -- One more test for "manual" validation of FKs, this time we rewrite the PK -- table. The NOT VALID constraint cannot be used if the FK table is -- partitioned and if PG version is < 18, so we need a separate test. CREATE TABLE tab1_pk(i int primary key); INSERT INTO tab1_pk(i) VALUES (1); CREATE TABLE tab1_pk_new(i bigint primary key); DROP TABLE tab1_fk; CREATE TABLE tab1_fk(i int REFERENCES tab1_pk); INSERT INTO tab1_fk(i) VALUES (1); \d tab1_pk SELECT rewrite_table('tab1_pk', 'tab1_pk_new', 'tab1_pk_orig'); \d tab1_pk ALTER TABLE tab1_fk VALIDATE CONSTRAINT tab1_fk_i_fkey2; \d tab1_pk -- For the partitioned FK table, test at least that the FK creation is skipped -- (i.e. ERROR saying that NOT VALID is not supported is no raised) DROP TABLE tab1_fk; CREATE TABLE tab1_fk(i int REFERENCES tab1_pk) PARTITION BY RANGE (i); CREATE TABLE tab1_fk_1 PARTITION OF tab1_fk DEFAULT; INSERT INTO tab1_fk(i) VALUES (1); ALTER TABLE tab1_pk_orig RENAME TO tab1_pk_new; TRUNCATE TABLE tab1_pk_new; \d tab1_fk SELECT rewrite_table('tab1_pk', 'tab1_pk_new', 'tab1_pk_orig'); -- Note that tab1_fk still references tab1_pk_orig - that's expected. \d tab1_fk -- The same once again, but now rewrite the FK table. DROP TABLE tab1_fk; DROP TABLE tab1_pk; ALTER TABLE tab1_pk_orig RENAME TO tab1_pk; CREATE TABLE tab1_fk(i int PRIMARY KEY REFERENCES tab1_pk); INSERT INTO tab1_fk(i) VALUES (1); CREATE TABLE tab1_fk_new(i int PRIMARY KEY) PARTITION BY RANGE (i); CREATE TABLE tab1_fk_new_1 PARTITION OF tab1_fk_new DEFAULT; \d tab1_fk SELECT rewrite_table('tab1_fk', 'tab1_fk_new', 'tab1_fk_orig'); \d tab1_fk -- Check if sequence on the target table is synchronized with that of the -- source table. CREATE TABLE tab5(i int primary key generated always as identity); CREATE TABLE tab5_new(i int primary key generated always as identity); INSERT INTO tab5(i) VALUES (DEFAULT); SELECT rewrite_table('tab5', 'tab5_new', 'tab5_orig'); INSERT INTO tab5(i) VALUES (DEFAULT); SELECT i FROM tab5 ORDER BY i; -- The same with serial column. CREATE TABLE tab6(i serial primary key); CREATE TABLE tab6_new(i serial primary key); INSERT INTO tab6(i) VALUES (DEFAULT); SELECT rewrite_table('tab6', 'tab6_new', 'tab6_orig'); INSERT INTO tab6(i) VALUES (DEFAULT); SELECT i FROM tab6 ORDER BY i; pg_rewrite-REL2_0_0/typedefs.list000066400000000000000000000002351504513220300171000ustar00rootroot00000000000000CatalogState ConcurrentChange ConcurrentChangeKind ConstraintInfo DecodingOutputState IndexCatInfo PartitionEntry PgClassCatInfo TypeCatInfo partitions_hash