libcudf  23.12.00
io/json.hpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020-2023, NVIDIA CORPORATION.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include "types.hpp"
20 
22 #include <cudf/types.hpp>
23 
25 
26 #include <map>
27 #include <string>
28 #include <variant>
29 #include <vector>
30 
31 namespace cudf {
32 namespace io {
39 class json_reader_options_builder;
40 
50 
54  std::map<std::string, schema_element> child_types;
55 };
56 
61  FAIL,
63 };
64 
89  source_info _source;
90 
91  // Data types of the column; empty to infer dtypes
92  std::variant<std::vector<data_type>,
93  std::map<std::string, data_type>,
94  std::map<std::string, schema_element>>
95  _dtypes;
96  // Specify the compression format of the source or infer from file extension
98 
99  // Read the file as a json object per line
100  bool _lines = false;
101 
102  // Bytes to skip from the start
103  size_t _byte_range_offset = 0;
104  // Bytes to read; always reads complete rows
105  size_t _byte_range_size = 0;
106 
107  // Whether to parse dates as DD/MM versus MM/DD
108  bool _dayfirst = false;
109 
110  // Whether to use the legacy reader
111  bool _legacy = false;
112 
113  // Whether to keep the quote characters of string values
114  bool _keep_quotes = false;
115 
116  // Whether to recover after an invalid JSON line
118 
124  explicit json_reader_options(source_info src) : _source{std::move(src)} {}
125 
127 
128  public:
134  json_reader_options() = default;
135 
143 
149  [[nodiscard]] source_info const& get_source() const { return _source; }
150 
156  std::variant<std::vector<data_type>,
157  std::map<std::string, data_type>,
158  std::map<std::string, schema_element>> const&
159  get_dtypes() const
160  {
161  return _dtypes;
162  }
163 
169  compression_type get_compression() const { return _compression; }
170 
176  size_t get_byte_range_offset() const { return _byte_range_offset; }
177 
183  size_t get_byte_range_size() const { return _byte_range_size; }
184 
191  {
192  if (_byte_range_size == 0) {
193  return 0;
194  } else {
195  return _byte_range_size + get_byte_range_padding();
196  }
197  }
198 
204  size_t get_byte_range_padding() const
205  {
206  auto const num_columns = std::visit([](auto const& dtypes) { return dtypes.size(); }, _dtypes);
207 
208  auto const max_row_bytes = 16 * 1024; // 16KB
209  auto const column_bytes = 64;
210  auto const base_padding = 1024; // 1KB
211 
212  if (num_columns == 0) {
213  // Use flat size if the number of columns is not known
214  return max_row_bytes;
215  }
216 
217  // Expand the size based on the number of columns, if available
218  return base_padding + num_columns * column_bytes;
219  }
220 
226  bool is_enabled_lines() const { return _lines; }
227 
233  bool is_enabled_dayfirst() const { return _dayfirst; }
234 
240  bool is_enabled_legacy() const { return _legacy; }
241 
247  bool is_enabled_keep_quotes() const { return _keep_quotes; }
248 
254  json_recovery_mode_t recovery_mode() const { return _recovery_mode; }
255 
261  void set_dtypes(std::vector<data_type> types) { _dtypes = std::move(types); }
262 
268  void set_dtypes(std::map<std::string, data_type> types) { _dtypes = std::move(types); }
269 
275  void set_dtypes(std::map<std::string, schema_element> types) { _dtypes = std::move(types); }
276 
282  void set_compression(compression_type comp_type) { _compression = comp_type; }
283 
289  void set_byte_range_offset(size_type offset) { _byte_range_offset = offset; }
290 
296  void set_byte_range_size(size_type size) { _byte_range_size = size; }
297 
303  void enable_lines(bool val) { _lines = val; }
304 
310  void enable_dayfirst(bool val) { _dayfirst = val; }
311 
317  void enable_legacy(bool val) { _legacy = val; }
318 
325  void enable_keep_quotes(bool val) { _keep_quotes = val; }
326 
332  void set_recovery_mode(json_recovery_mode_t val) { _recovery_mode = val; }
333 };
334 
339  json_reader_options options;
340 
341  public:
347  explicit json_reader_options_builder() = default;
348 
354  explicit json_reader_options_builder(source_info src) : options{std::move(src)} {}
355 
362  json_reader_options_builder& dtypes(std::vector<data_type> types)
363  {
364  options._dtypes = std::move(types);
365  return *this;
366  }
367 
374  json_reader_options_builder& dtypes(std::map<std::string, data_type> types)
375  {
376  options._dtypes = std::move(types);
377  return *this;
378  }
379 
386  json_reader_options_builder& dtypes(std::map<std::string, schema_element> types)
387  {
388  options._dtypes = std::move(types);
389  return *this;
390  }
391 
399  {
400  options._compression = comp_type;
401  return *this;
402  }
403 
411  {
412  options._byte_range_offset = offset;
413  return *this;
414  }
415 
423  {
424  options._byte_range_size = size;
425  return *this;
426  }
427 
435  {
436  options._lines = val;
437  return *this;
438  }
439 
447  {
448  options._dayfirst = val;
449  return *this;
450  }
451 
459  {
460  options._legacy = val;
461  return *this;
462  }
463 
472  {
473  options._keep_quotes = val;
474  return *this;
475  }
476 
484  {
485  options._recovery_mode = val;
486  return *this;
487  }
488 
492  operator json_reader_options&&() { return std::move(options); }
493 
501  json_reader_options&& build() { return std::move(options); }
502 };
503 
522  json_reader_options options,
525  // end of group
527 
538 
543  // Specify the sink to use for writer output
544  sink_info _sink;
545  // Set of columns to output
546  table_view _table;
547  // string to use for null entries
548  std::string _na_rep = "";
549  // Indicates whether to output nulls as 'null' or exclude the field
550  bool _include_nulls = false;
551  // Indicates whether to use JSON lines for records format
552  bool _lines = false;
553  // maximum number of rows to write in each chunk (limits memory use)
554  size_type _rows_per_chunk = std::numeric_limits<size_type>::max();
555  // string to use for values != 0 in INT8 types (default 'true')
556  std::string _true_value = std::string{"true"};
557  // string to use for values == 0 in INT8 types (default 'false')
558  std::string _false_value = std::string{"false"};
559  // Names of all columns; if empty, writer will generate column names
560  std::optional<table_metadata> _metadata; // Optional column names
561 
568  explicit json_writer_options(sink_info const& sink, table_view const& table)
569  : _sink(sink), _table(table), _rows_per_chunk(table.num_rows())
570  {
571  }
572 
574 
575  public:
581  explicit json_writer_options() = default;
582 
592 
598  [[nodiscard]] sink_info const& get_sink() const { return _sink; }
599 
605  [[nodiscard]] table_view const& get_table() const { return _table; }
606 
612  [[nodiscard]] std::optional<table_metadata> const& get_metadata() const { return _metadata; }
613 
619  [[nodiscard]] std::string const& get_na_rep() const { return _na_rep; }
620 
626  [[nodiscard]] bool is_enabled_include_nulls() const { return _include_nulls; }
627 
633  [[nodiscard]] bool is_enabled_lines() const { return _lines; }
634 
640  [[nodiscard]] size_type get_rows_per_chunk() const { return _rows_per_chunk; }
641 
647  [[nodiscard]] std::string const& get_true_value() const { return _true_value; }
648 
654  [[nodiscard]] std::string const& get_false_value() const { return _false_value; }
655 
656  // Setter
657 
663  void set_table(table_view tbl) { _table = tbl; }
664 
670  void set_metadata(table_metadata metadata) { _metadata = std::move(metadata); }
671 
677  void set_na_rep(std::string val) { _na_rep = std::move(val); }
678 
684  void enable_include_nulls(bool val) { _include_nulls = val; }
685 
691  void enable_lines(bool val) { _lines = val; }
692 
698  void set_rows_per_chunk(size_type val) { _rows_per_chunk = val; }
699 
705  void set_true_value(std::string val) { _true_value = std::move(val); }
706 
712  void set_false_value(std::string val) { _false_value = std::move(val); }
713 };
714 
719  json_writer_options options;
720 
721  public:
727  explicit json_writer_options_builder() = default;
728 
736  : options{sink, table}
737  {
738  }
739 
747  {
748  options._table = tbl;
749  return *this;
750  }
751 
759  {
760  options._metadata = std::move(metadata);
761  return *this;
762  }
763 
771  {
772  options._na_rep = std::move(val);
773  return *this;
774  };
775 
783  {
784  options._include_nulls = val;
785  return *this;
786  }
787 
795  {
796  options._lines = val;
797  return *this;
798  }
799 
807  {
808  options._rows_per_chunk = val;
809  return *this;
810  }
811 
819  {
820  options._true_value = std::move(val);
821  return *this;
822  }
823 
831  {
832  options._false_value = std::move(val);
833  return *this;
834  }
835 
839  operator json_writer_options&&() { return std::move(options); }
840 
848  json_writer_options&& build() { return std::move(options); }
849 };
850 
869 void write_json(json_writer_options const& options,
872  // end of group
874 } // namespace io
875 } // namespace cudf
Indicator for the logical data type of an element in a column.
Definition: types.hpp:227
Builds settings to use for read_json().
Definition: io/json.hpp:338
json_reader_options_builder & keep_quotes(bool val)
Set whether the reader should keep quotes of string values.
Definition: io/json.hpp:471
json_reader_options_builder & dayfirst(bool val)
Set whether to parse dates as DD/MM versus MM/DD.
Definition: io/json.hpp:446
json_reader_options_builder & recovery_mode(json_recovery_mode_t val)
Specifies the JSON reader's behavior on invalid JSON lines.
Definition: io/json.hpp:483
json_reader_options_builder & lines(bool val)
Set whether to read the file as a json object per line.
Definition: io/json.hpp:434
json_reader_options_builder & dtypes(std::vector< data_type > types)
Set data types for columns to be read.
Definition: io/json.hpp:362
json_reader_options && build()
move json_reader_options member once it's built.
Definition: io/json.hpp:501
json_reader_options_builder & compression(compression_type comp_type)
Set the compression type.
Definition: io/json.hpp:398
json_reader_options_builder(source_info src)
Constructor from source info.
Definition: io/json.hpp:354
json_reader_options_builder & byte_range_size(size_type size)
Set number of bytes to read.
Definition: io/json.hpp:422
json_reader_options_builder & dtypes(std::map< std::string, schema_element > types)
Set data types for columns to be read.
Definition: io/json.hpp:386
json_reader_options_builder & legacy(bool val)
Set whether to use the legacy reader.
Definition: io/json.hpp:458
json_reader_options_builder & byte_range_offset(size_type offset)
Set number of bytes to skip from source start.
Definition: io/json.hpp:410
json_reader_options_builder()=default
Default constructor.
json_reader_options_builder & dtypes(std::map< std::string, data_type > types)
Set data types for columns to be read.
Definition: io/json.hpp:374
Input arguments to the read_json interface.
Definition: io/json.hpp:88
void set_compression(compression_type comp_type)
Set the compression type.
Definition: io/json.hpp:282
void set_dtypes(std::vector< data_type > types)
Set data types for columns to be read.
Definition: io/json.hpp:261
bool is_enabled_keep_quotes() const
Whether the reader should keep quotes of string values.
Definition: io/json.hpp:247
size_t get_byte_range_offset() const
Returns number of bytes to skip from source start.
Definition: io/json.hpp:176
source_info const & get_source() const
Returns source info.
Definition: io/json.hpp:149
void enable_legacy(bool val)
Set whether to use the legacy reader.
Definition: io/json.hpp:317
void set_dtypes(std::map< std::string, data_type > types)
Set data types for columns to be read.
Definition: io/json.hpp:268
bool is_enabled_lines() const
Whether to read the file as a json object per line.
Definition: io/json.hpp:226
void set_byte_range_size(size_type size)
Set number of bytes to read.
Definition: io/json.hpp:296
json_reader_options()=default
Default constructor.
void enable_dayfirst(bool val)
Set whether to parse dates as DD/MM versus MM/DD.
Definition: io/json.hpp:310
std::variant< std::vector< data_type >, std::map< std::string, data_type >, std::map< std::string, schema_element > > const & get_dtypes() const
Returns data types of the columns.
Definition: io/json.hpp:159
size_t get_byte_range_size_with_padding() const
Returns number of bytes to read with padding.
Definition: io/json.hpp:190
void set_recovery_mode(json_recovery_mode_t val)
Specifies the JSON reader's behavior on invalid JSON lines.
Definition: io/json.hpp:332
void enable_lines(bool val)
Set whether to read the file as a json object per line.
Definition: io/json.hpp:303
void enable_keep_quotes(bool val)
Set whether the reader should keep quotes of string values.
Definition: io/json.hpp:325
compression_type get_compression() const
Returns compression format of the source.
Definition: io/json.hpp:169
void set_dtypes(std::map< std::string, schema_element > types)
Set data types for a potentially nested column hierarchy.
Definition: io/json.hpp:275
size_t get_byte_range_size() const
Returns number of bytes to read.
Definition: io/json.hpp:183
json_recovery_mode_t recovery_mode() const
Queries the JSON reader's behavior on invalid JSON lines.
Definition: io/json.hpp:254
static json_reader_options_builder builder(source_info src)
create json_reader_options_builder which will build json_reader_options.
void set_byte_range_offset(size_type offset)
Set number of bytes to skip from source start.
Definition: io/json.hpp:289
bool is_enabled_legacy() const
Whether the legacy reader should be used.
Definition: io/json.hpp:240
bool is_enabled_dayfirst() const
Whether to parse dates as DD/MM versus MM/DD.
Definition: io/json.hpp:233
size_t get_byte_range_padding() const
Returns number of bytes to pad when reading.
Definition: io/json.hpp:204
Builder to build options for writer_json()
Definition: io/json.hpp:718
json_writer_options_builder & include_nulls(bool val)
Enables/Disables output of nulls as 'null'.
Definition: io/json.hpp:782
json_writer_options_builder & table(table_view tbl)
Sets table to be written to output.
Definition: io/json.hpp:746
json_writer_options_builder()=default
Default constructor.
json_writer_options_builder & rows_per_chunk(int val)
Sets maximum number of rows to process for each file write.
Definition: io/json.hpp:806
json_writer_options_builder & true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: io/json.hpp:818
json_writer_options_builder & false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: io/json.hpp:830
json_writer_options_builder(sink_info const &sink, table_view const &table)
Constructor from sink and table.
Definition: io/json.hpp:735
json_writer_options_builder & na_rep(std::string val)
Sets string to used for null entries.
Definition: io/json.hpp:770
json_writer_options_builder & metadata(table_metadata metadata)
Sets optional metadata (with column names).
Definition: io/json.hpp:758
json_writer_options && build()
move json_writer_options member once it's built.
Definition: io/json.hpp:848
json_writer_options_builder & lines(bool val)
Enables/Disables JSON lines for records format.
Definition: io/json.hpp:794
Settings to use for write_json().
Definition: io/json.hpp:542
table_view const & get_table() const
Returns table that would be written to output.
Definition: io/json.hpp:605
void set_false_value(std::string val)
Sets string used for values == 0 in INT8 types.
Definition: io/json.hpp:712
void enable_include_nulls(bool val)
Enables/Disables output of nulls as 'null'.
Definition: io/json.hpp:684
bool is_enabled_include_nulls() const
Whether to output nulls as 'null'.
Definition: io/json.hpp:626
void enable_lines(bool val)
Enables/Disables JSON lines for records format.
Definition: io/json.hpp:691
void set_na_rep(std::string val)
Sets string to used for null entries.
Definition: io/json.hpp:677
static json_writer_options_builder builder(sink_info const &sink, table_view const &table)
Create builder to create json_writer_options.
json_writer_options()=default
Default constructor.
void set_true_value(std::string val)
Sets string used for values != 0 in INT8 types.
Definition: io/json.hpp:705
sink_info const & get_sink() const
Returns sink used for writer output.
Definition: io/json.hpp:598
void set_rows_per_chunk(size_type val)
Sets maximum number of rows to process for each file write.
Definition: io/json.hpp:698
std::string const & get_true_value() const
Returns string used for values != 0 in INT8 types.
Definition: io/json.hpp:647
void set_table(table_view tbl)
Sets table to be written to output.
Definition: io/json.hpp:663
std::string const & get_false_value() const
Returns string used for values == 0 in INT8 types.
Definition: io/json.hpp:654
bool is_enabled_lines() const
Whether to use JSON lines for records format.
Definition: io/json.hpp:633
size_type get_rows_per_chunk() const
Returns maximum number of rows to process for each file write.
Definition: io/json.hpp:640
std::optional< table_metadata > const & get_metadata() const
Returns metadata information.
Definition: io/json.hpp:612
std::string const & get_na_rep() const
Returns string to used for null entries.
Definition: io/json.hpp:619
void set_metadata(table_metadata metadata)
Sets metadata.
Definition: io/json.hpp:670
A set of cudf::column_view's of the same size.
Definition: table_view.hpp:187
A set of cudf::column's of the same size.
Definition: table.hpp:40
size_type num_rows() const noexcept
Returns the number of rows.
Definition: table.hpp:87
table_with_metadata read_json(json_reader_options options, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::mr::device_memory_resource *mr=rmm::mr::get_current_device_resource())
Reads a JSON dataset into a set of columns.
json_recovery_mode_t
Control the error recovery behavior of the json parser.
Definition: io/json.hpp:60
@ RECOVER_WITH_NULL
Recovers from an error, replacing invalid records with null.
@ FAIL
Does not recover from an error when encountering an invalid format.
void write_json(json_writer_options const &options, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::mr::device_memory_resource *mr=rmm::mr::get_current_device_resource())
Writes a set of columns to JSON format.
device_memory_resource * get_current_device_resource()
int32_t size_type
Row index type for columns and tables.
Definition: types.hpp:80
compression_type
Compression algorithms.
Definition: io/types.hpp:50
@ AUTO
Automatically detect or select compression format.
cuDF interfaces
Definition: aggregation.hpp:34
rmm::cuda_stream_view const get_default_stream()
Get the current default stream.
Allows specifying the target types for nested JSON data via json_reader_options' set_dtypes method.
Definition: io/json.hpp:45
data_type type
The type that this column should be converted to.
Definition: io/json.hpp:49
std::map< std::string, schema_element > child_types
Allows specifying this column's child columns target type.
Definition: io/json.hpp:54
Destination information for write interfaces.
Definition: io/types.hpp:463
Source information for read interfaces.
Definition: io/types.hpp:288
Table metadata returned by IO readers.
Definition: io/types.hpp:231
Table with table metadata used by io readers to return the metadata by value.
Definition: io/types.hpp:243
Class definitions for (mutable)_table_view
Type declarations for libcudf.