org.apache.parquet.hadoop

Class ParquetFileWriter

java.lang.Object

org.apache.parquet.hadoop.ParquetFileWriter

public class ParquetFileWriter
extends Object

Internal implementation of the Parquet file writer as a block container

Author:

Julien Le Dem

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	ParquetFileWriter.Mode

Field Summary

Fields

Modifier and Type	Field and Description
static int	CURRENT_VERSION
static byte[]	MAGIC
static String	PARQUET_COMMON_METADATA_FILE
static String	PARQUET_METADATA_FILE

Constructor Summary

Constructors

Constructor and Description
ParquetFileWriter(org.apache.hadoop.conf.Configuration configuration, MessageType schema, org.apache.hadoop.fs.Path file)
ParquetFileWriter(org.apache.hadoop.conf.Configuration configuration, MessageType schema, org.apache.hadoop.fs.Path file, ParquetFileWriter.Mode mode)
ParquetFileWriter(org.apache.hadoop.conf.Configuration configuration, MessageType schema, org.apache.hadoop.fs.Path file, ParquetFileWriter.Mode mode, long rowGroupSize, int maxPaddingSize)

Method Summary

Methods

Modifier and Type	Method and Description
void	appendFile(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.Path file)
void	appendRowGroup(org.apache.hadoop.fs.FSDataInputStream from, BlockMetaData rowGroup, boolean dropColumns)
void	appendRowGroup(SeekableInputStream from, BlockMetaData rowGroup, boolean dropColumns)
void	appendRowGroups(org.apache.hadoop.fs.FSDataInputStream file, List<BlockMetaData> rowGroups, boolean dropColumns)
void	appendRowGroups(SeekableInputStream file, List<BlockMetaData> rowGroups, boolean dropColumns)
void	end(Map<String,String> extraMetaData) ends a file once all blocks have been written.
void	endBlock() ends a block once all column chunks have been written
void	endColumn() end a column (once all rep, def and data have been written)
long	getNextRowGroupSize()
long	getPos()
void	start() start the file
void	startBlock(long recordCount) start a block
void	startColumn(ColumnDescriptor descriptor, long valueCount, CompressionCodecName compressionCodecName) start a column inside a block
void	writeDataPage(int valueCount, int uncompressedPageSize, org.apache.parquet.bytes.BytesInput bytes, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding) Deprecated.
void	writeDataPage(int valueCount, int uncompressedPageSize, org.apache.parquet.bytes.BytesInput bytes, Statistics statistics, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding) writes a single page
void	writeDictionaryPage(DictionaryPage dictionaryPage) writes a dictionary page page
static void	writeMetadataFile(org.apache.hadoop.conf.Configuration configuration, org.apache.hadoop.fs.Path outputPath, List<Footer> footers) writes a _metadata and _common_metadata file

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

PARQUET_METADATA_FILE

public static final String PARQUET_METADATA_FILE

See Also:

Constant Field Values

PARQUET_COMMON_METADATA_FILE

public static final String PARQUET_COMMON_METADATA_FILE

See Also:

Constant Field Values

MAGIC

public static final byte[] MAGIC

CURRENT_VERSION

public static final int CURRENT_VERSION

See Also:

Constant Field Values

Constructor Detail

ParquetFileWriter

public ParquetFileWriter(org.apache.hadoop.conf.Configuration configuration,
                 MessageType schema,
                 org.apache.hadoop.fs.Path file)
                  throws IOException

Parameters:

configuration - Hadoop configuration

schema - the schema of the data

file - the file to write to

Throws:

IOException - if the file can not be created

ParquetFileWriter

public ParquetFileWriter(org.apache.hadoop.conf.Configuration configuration,
                 MessageType schema,
                 org.apache.hadoop.fs.Path file,
                 ParquetFileWriter.Mode mode)
                  throws IOException

Parameters:

configuration - Hadoop configuration

schema - the schema of the data

file - the file to write to

mode - file creation mode

Throws:

IOException - if the file can not be created

ParquetFileWriter

public ParquetFileWriter(org.apache.hadoop.conf.Configuration configuration,
                 MessageType schema,
                 org.apache.hadoop.fs.Path file,
                 ParquetFileWriter.Mode mode,
                 long rowGroupSize,
                 int maxPaddingSize)
                  throws IOException

Parameters:

configuration - Hadoop configuration

schema - the schema of the data

file - the file to write to

mode - file creation mode

rowGroupSize - the row group size

Throws:

IOException - if the file can not be created

Method Detail

start

public void start()
           throws IOException

start the file

Throws:

IOException

startBlock

public void startBlock(long recordCount)
                throws IOException

start a block

Parameters:

recordCount - the record count in this block

Throws:

IOException

startColumn

public void startColumn(ColumnDescriptor descriptor,
               long valueCount,
               CompressionCodecName compressionCodecName)
                 throws IOException

start a column inside a block

Parameters:

descriptor - the column descriptor

valueCount - the value count in this column

compressionCodecName -

Throws:

IOException

writeDictionaryPage

public void writeDictionaryPage(DictionaryPage dictionaryPage)
                         throws IOException

writes a dictionary page page

Parameters:

dictionaryPage - the dictionary page

Throws:

IOException

writeDataPage

@Deprecated
public void writeDataPage(int valueCount,
                            int uncompressedPageSize,
                            org.apache.parquet.bytes.BytesInput bytes,
                            Encoding rlEncoding,
                            Encoding dlEncoding,
                            Encoding valuesEncoding)
                   throws IOException

Deprecated.

writes a single page

Parameters:

valueCount - count of values

uncompressedPageSize - the size of the data once uncompressed

bytes - the compressed data for the page without header

rlEncoding - encoding of the repetition level

dlEncoding - encoding of the definition level

valuesEncoding - encoding of values

Throws:

IOException

writeDataPage

public void writeDataPage(int valueCount,
                 int uncompressedPageSize,
                 org.apache.parquet.bytes.BytesInput bytes,
                 Statistics statistics,
                 Encoding rlEncoding,
                 Encoding dlEncoding,
                 Encoding valuesEncoding)
                   throws IOException

writes a single page

Parameters:

valueCount - count of values

uncompressedPageSize - the size of the data once uncompressed

bytes - the compressed data for the page without header

rlEncoding - encoding of the repetition level

dlEncoding - encoding of the definition level

valuesEncoding - encoding of values

Throws:

IOException

endColumn

public void endColumn()
               throws IOException

end a column (once all rep, def and data have been written)

Throws:

IOException

endBlock

public void endBlock()
              throws IOException

ends a block once all column chunks have been written

Throws:

IOException

appendFile

public void appendFile(org.apache.hadoop.conf.Configuration conf,
              org.apache.hadoop.fs.Path file)
                throws IOException

Throws:

IOException

appendRowGroups

public void appendRowGroups(org.apache.hadoop.fs.FSDataInputStream file,
                   List<BlockMetaData> rowGroups,
                   boolean dropColumns)
                     throws IOException

Throws:

IOException

appendRowGroups

public void appendRowGroups(SeekableInputStream file,
                   List<BlockMetaData> rowGroups,
                   boolean dropColumns)
                     throws IOException

Throws:

IOException

appendRowGroup

public void appendRowGroup(org.apache.hadoop.fs.FSDataInputStream from,
                  BlockMetaData rowGroup,
                  boolean dropColumns)
                    throws IOException

Throws:

IOException

appendRowGroup

public void appendRowGroup(SeekableInputStream from,
                  BlockMetaData rowGroup,
                  boolean dropColumns)
                    throws IOException

Throws:

IOException

end

public void end(Map<String,String> extraMetaData)
         throws IOException

ends a file once all blocks have been written. closes the file.

Parameters:

extraMetaData - the extra meta data to write in the footer

Throws:

IOException

writeMetadataFile

public static void writeMetadataFile(org.apache.hadoop.conf.Configuration configuration,
                     org.apache.hadoop.fs.Path outputPath,
                     List<Footer> footers)
                              throws IOException

writes a _metadata and _common_metadata file

Parameters:

configuration - the configuration to use to get the FileSystem

outputPath - the directory to write the _metadata file to

footers - the list of footers to merge

Throws:

IOException

getPos

public long getPos()
            throws IOException

Returns:

the current position in the underlying file

Throws:

IOException

getNextRowGroupSize

public long getNextRowGroupSize()
                         throws IOException

Throws:

IOException