src/include/cdb/cdbappendonlystoragewrite.h

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*-------------------------------------------------------------------------
 *
 * cdbappendonlystoragewrite.h
 *
 *-------------------------------------------------------------------------
 */
#ifndef CDBAPPENDONLYSTORAGEWRITE_H
#define CDBAPPENDONLYSTORAGEWRITE_H

#include "catalog/pg_compression.h"
#include "cdb/cdbappendonlystorage.h"
#include "cdb/cdbappendonlystoragelayer.h"
#include "cdb/cdbbufferedappend.h"
#include "utils/palloc.h"
#include "storage/fd.h"



/*
 * This structure contains write session information.  Consider the fields
 * inside to be private.
 */
typedef struct AppendOnlyStorageWrite
{
	bool	isActive;

	MemoryContext memoryContext;
			/*
			 * The memory context to use for buffers and
			 * other memory needs.
			 */

    int32         maxBufferLen;
			/*
			 * The maximum Append-Only Storage Block
			 * length including all storage headers.
			 */

	int32		largeWriteLen;
			/*
			 * The large write length given to the BufferedAppend module.
			 */

	char		 *relationName;
			/*
			 * Name of the relation to use in system
			 * logging and error messages.
			 */

	char		*title;
			/*
			 * A phrase that better describes the purpose of the this open.
			 *
			 * The caller manages the storage for this.
			 */
			 
	AppendOnlyStorageAttributes		storageAttributes;
			/*
			 * The Append-Only Storage Attributes
			 * from relation creation.
			 */

	int32			regularHeaderLen;
			/*
			 * Fixed header length as determined by the checksum flag in
			 * storageAttributes.
			 */

	BufferedAppend	bufferedAppend;
			/*
			 * The BufferedAppend module's object that holds write session data.
			 */

	char		 *segmentFileName;
			/*
			 * Name of the current segment file name to use in system
			 * logging and error messages.
			 */

	File	file;
			/*
			 * The handle to the current open segment file.
			 */

	int64	startEof;
			/*
			 * The committed EOF at the beginning of the write open.
			 */

	RelFileNode	relFileNode;
	int32		segmentFileNum;

/*	ItemPointerData 	persistentTid;
	int64				persistentSerialNum;*/
			/*
			 * Persistence information for current write open.
			 */

	int64	bufferCount;
			/*
			 * The number of blocks written since the beginning of the segment file.
			 */

	int64   lastWriteBeginPosition;
			/* The beginning of the write buffer for the last write. */

	AoHeaderKind	getBufferAoHeaderKind;
			/* The kind of header specifed to ~_GetBuffer. */

	int32	currentCompleteHeaderLen;
			/*
			 * Complete header length of the current block.  Varies depending on whether the
			 * first row number was set.
			 */

	uint8	*currentBuffer;
			 /*
			  * Current block pointer within the BufferedAppend buffer.
			  */

	uint8	*uncompressedBuffer;
			 /*
			  * A temporary buffer that is given back from *_GetBuffer when
			  * compression is being done.
			  */

	int32	compressionOverrunLen;
			/*
			 * Number of bytes of extra buffer must have beyond the output
			 * compression buffer needed for spillover for some compression libraries.
			 */

	int32	maxBufferWithCompressionOverrrunLen;
			/* The maximum buffer plus compression overrun byte length */

	uint8	*verifyWriteBuffer;
			/*
			 * When non-null, we are doing VerifyBlock and this
			 * is the buffer to decompress the just compressed block
			 * into so we can memory comprare it with the input.
			 */
	/*
	 * Add these two byte lengths to get the length of the qlzScratchDecompress buffer.
	 */

	/* Storage attributes */
	CompressionState *compressionState;
	CompressionState *verifyWriteCompressionState; /* This is only valid if the gp_appendonly_verify_write_block GUC is set. */
	int               blocksize;                   /* For AO or CO uncompresed block size         */
	PGFunction       *compression_functions;       /* For AO or CO compression.                   */
			/* The array index corresponds to COMP_FUNC_*  */

} AppendOnlyStorageWrite;

// -----------------------------------------------------------------------------
// Initialization
// -----------------------------------------------------------------------------

/*
 * Initialize AppendOnlyStorageWrite.
 *
 * The AppendOnlyStorageWrite data structure is initialized
 * once for a append �session� and can be used to add
 * Append-Only Storage Blocks to 1 or more segment files.
 *
 * The current file to write to is opened with the
 * AppendOnlyStorageWrite_OpenFile routine.
*/
extern void AppendOnlyStorageWrite_Init(
    AppendOnlyStorageWrite			*storageWrite,
				/* The data structure to initialize. */

	MemoryContext 					memoryContext,
				/*
				 * The memory context to use for buffers and
				 * other memory needs.  When NULL, the
				 * current memory context is used.
				 */
    int32                			maxBufferLen,
				/*
				 * The maximum Append-Only Storage Block
				 * length including all storage headers.
				 */
	char							*relationName,
				/*
				 * Name of the relation to use in system
				 * logging and error messages.
				 */

	char							*title,
				/*
				 * A phrase that better describes the purpose of the this open.
				 *
				 * The caller manages the storage for this.
				 */

	AppendOnlyStorageAttributes		*storageAttributes);
				/*
				 * The Append-Only Storage Attributes
				 * from relation creation.
				 */

/*
 * Return (read-only) pointer to relation name.
 */
extern char* AppendOnlyStorageWrite_RelationName(
	AppendOnlyStorageWrite			*storageWrite);


/*
 * Finish using the AppendOnlyStorageWrite session created with ~Init.
 */
extern void AppendOnlyStorageWrite_FinishSession(
	AppendOnlyStorageWrite			*storageWrite);
				/* The data structure to finish. */

// -----------------------------------------------------------------------------
// Open and FlushAndClose
// -----------------------------------------------------------------------------

/*
 * Creates an on-demand Append-Only segment file under transaction.
 */
void AppendOnlyStorageWrite_TransactionCreateFile(
	char						*relname,

	int64						logicalEof,
				/*
				 * The last committed write transaction's EOF
				 * value to use as the end of the segment
				 * file.
				 *
				 * If the EOF is 0, we will create the file
				 * if necessary.  Otherwise, it must already
				 * exist.
				 */
	RelFileNode 				*relFileNode,
	int32						segmentFileNum,
	ItemPointer 				persistentTid,
	int64						*persistentSerialNum);

/*
 * Opens the next segment file to write.  The file must already exist.
 *
 * This routine is responsible for seeking to the proper
 * write location given the logical EOF.
 */
extern void AppendOnlyStorageWrite_OpenFile(
	AppendOnlyStorageWrite		*storageWrite,

    char				 		*filePathName,
				/* The name of the segment file to open. */

    int64				 		logicalEof,
				/*
				 * The last committed write transaction�s EOF
				 * value to use as the end of the segment
				 * file.
				 *
				 * If the EOF is 0, we will create the file
				 * if necessary.  Otherwise, it must already
				 * exist.
				 */
	int64						fileLen_uncompressed,
	RelFileNode					*relFileNode,
	int32						segmentFileNum
	/*ItemPointer 				persistentTid,
	int64						persistentSerialNum*/);

/*
 * Flush and close the current segment file.
 *
 * No error if the current is already closed.
 */
extern void AppendOnlyStorageWrite_FlushAndCloseFile(
	AppendOnlyStorageWrite		*storageWrite,

	int64						*newLogicalEof,
				/* The new EOF for the segment file. */

	int64						*fileLen_uncompressed);

/*
 * Flush and close the current segment file under a transaction.
 *
 * Handles mirror loss end transaction work.
 *
 * No error if the current is already closed.
 */
void AppendOnlyStorageWrite_TransactionFlushAndCloseFile(
	AppendOnlyStorageWrite		*storageWrite,

	int64						*newLogicalEof,
				/* The new EOF for the segment file. */

	int64						*fileLen_uncompressed);

// -----------------------------------------------------------------------------
// Usable Block Length
// -----------------------------------------------------------------------------

/*
 *    When writing �short� content intended to stay within the maxBufferLen (also known as
 *    blocksize), some of the buffer will be used for the Append-Only Block Header.  This
 *    function returns that overhead length.
 *
 *    Isn�t the length of the Append-Only Storage Block constant? NO.
 *
 *    Currently, there are two things that can make it longer.  When checksums are configured,
 *    we add checksum data to the header.  And there is optional header data
 *   (e.g. firstRowNum).
 *
 *   We call the header portion with the optional checksums the fixed header because we
 *   need to be able to read and evaluate the checksums before we can interpret flags
 *   in the fixed header that indicate there is more header information.
 *
 *  The complete header length is the fixed header plus optional information.
 */

/*
 * Returns the Append-Only Storage Block fixed header length in bytes.
 */
extern int32 AppendOnlyStorageWrite_FixedHeaderLen(
	AppendOnlyStorageWrite		*storageWrite);

/*
 * Returns the Append-Only Storage Block complete header length in bytes.
 *
 * Call this routine after adding all optional header information for the current block
 * begin written.
 */
extern int32 AppendOnlyStorageWrite_CompleteHeaderLen(
	AppendOnlyStorageWrite		*storageWrite,
	AoHeaderKind				aoHeaderKind);

// -----------------------------------------------------------------------------
// Writing �Short� Content Efficiently that is not being Bulk Compressed
// -----------------------------------------------------------------------------

/*
 *    This section describes for writing content that is less than or equal to the blocksize
 *    (e.g. 32k) bytes that is not being bulk compressed by the Append-Only Storage Layer.
 *
 *    Actually, the content is limited to blocksize minus the Append-Only header size
 *    (see the AppendOnlyStorageWrite_HeaderLen routine).
 */


/*
 * Get a pointer to next maximum length buffer space for
 * appending �small� content.
 *
 * NOTE: The maximum length buffer space =
 *				maxBufferLen �
 *				AppendOnlyStorageWrite_HeaderLen(...)
 *
 * When compression is not being used, this interface provides a pointer directly into the
 * write buffer for efficient data generation.  Otherwise, a pointer to a temporary buffer
 * will be provided.
 *
 * Returns NULL when the current file does not have enough
 * room for another buffer.
 */
extern uint8 *AppendOnlyStorageWrite_GetBuffer(
	AppendOnlyStorageWrite		*storageWrite,
	int 						aoHeaderKind);

/*
 * Test if a buffer is currently allocated.
 */
extern bool AppendOnlyStorageWrite_IsBufferAllocated(
	AppendOnlyStorageWrite		*storageWrite);

/*
 * Return the beginning of the last write position of
 * the write buffer.
 */
extern int64 AppendOnlyStorageWrite_LastWriteBeginPosition(
	AppendOnlyStorageWrite *storageWrite);

/*
 * Return the position of the current write buffer.
 */
extern int64 AppendOnlyStorageWrite_CurrentPosition(
	AppendOnlyStorageWrite		*storageWrite);

/*
 * Return the internal current write buffer that includes the header.
 * UNDONE: Fix this interface privacy violation...
 */
extern uint8 *AppendOnlyStorageWrite_GetCurrentInternalBuffer(
		AppendOnlyStorageWrite		*storageWrite);

/*
 * Mark the current buffer "small" buffer as finished.
 *
 * If compression is configured, we will try to compress the contents in
 * the temporary uncompressed buffer into the write buffer.
 *
 * The buffer can be scheduled for writing and reused.
 */
extern void AppendOnlyStorageWrite_FinishBuffer(
	AppendOnlyStorageWrite		*storageWrite,

    int32                		contentLen,
			/*
			 * The byte length of the content generated
			 * directly into the buffer returned by
			 * AppendOnlyStorageWrite_GetBuffer.
			 */
	int							executorBlockKind,
			/*
			 * A value defined externally by the executor
			 * that describes in content stored in the
			 * Append-Only Storage Block.
			 */
	int							rowCount);
			/* The number of rows stored in the content. */

/*
 * Cancel the last ~GetBuffer call.
 *
 * This will also turn off the firstRowNum flag.
 */
extern void AppendOnlyStorageWrite_CancelLastBuffer(
	AppendOnlyStorageWrite		*storageWrite);


// -----------------------------------------------------------------------------
// Writing "Large" and/or Compressed Content
// -----------------------------------------------------------------------------

/*
 *    This section describes for writing long content that can be up to 1 Gb long and/or
 *    content that will be bulk-compressed when configured.
 */


/*
 * Write content up to 1Gb.
 *
 * Large content will be writen in fragment blocks by
 * the Append-Only Storage Layer.
 *
 * If compression is configured, then the content will be
 * compressed in fragments.
 *
 * Returns NULL when the current file does not have enough
 * room for another buffer.
 */
extern void AppendOnlyStorageWrite_Content(
	AppendOnlyStorageWrite		*storageWrite,

	uint8						*content,
			/* The content to store.  All contiguous. */

	int32                		contentLen,
			/* The byte length of the data to store. */

	int							executorBlockKind,
			/*
			 * A value defined externally by the executor
			 * that describes in content stored in the
			 * Append-Only Storage Block.
			 */
	int							rowCount);
			/* The number of rows stored in the content. */


// -----------------------------------------------------------------------------
// Optional: Set First Row Number
// -----------------------------------------------------------------------------

/*
 * Normally, the first row of an Append-Only Storage Block is implicit.  It is the last row
 * number of the previous block + 1.	However, to support BTree indicies that stored TIDs
 * in shared-memory/disk before the transaction commits, we may need to not reuse row
 * numbers of aborted transactions.  So, this routine tells the Append-Only Storage Layer
 * to explicitly keep the first row number.	This will take up more header overhead, so the
 * AppendOnlyStorageWrite_HeaderLen routine should be called afterwards to get the
 * new overhead length.
 */

extern char *AppendOnlyStorageWrite_ContextStr(
	AppendOnlyStorageWrite		*storageWrite);

#endif   /* CDBAPPENDONLYSTORAGEWRITE_H */