subword_tokenize.hpp

Go to the documentation of this file.

/*
 * Copyright (c) 2020-2023, NVIDIA CORPORATION.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once
 
#include <cudf/column/column.hpp>
#include <cudf/column/column_view.hpp>
#include <cudf/strings/strings_column_view.hpp>
 
namespace nvtext {
 
struct hashed_vocabulary {
  uint16_t first_token_id{};            
  uint16_t separator_token_id{};        
  uint16_t unknown_token_id{};          
  uint32_t outer_hash_a{};              
  uint32_t outer_hash_b{};              
  uint16_t num_bins{};                  
  std::unique_ptr<cudf::column> table;  
  std::unique_ptr<cudf::column> bin_coefficients;  
  std::unique_ptr<cudf::column> bin_offsets;  
  std::unique_ptr<cudf::column>
    cp_metadata;  
  std::unique_ptr<cudf::column>
    aux_cp_table;  
};
 
std::unique_ptr<hashed_vocabulary> load_vocabulary_file(
  std::string const& filename_hashed_vocabulary,
  rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
 
struct tokenizer_result {
  uint32_t nrows_tensor{};
  uint32_t sequence_length{};
  std::unique_ptr<cudf::column> tensor_token_ids;
  std::unique_ptr<cudf::column> tensor_attention_mask;
  std::unique_ptr<cudf::column> tensor_metadata;
};
 
tokenizer_result subword_tokenize(
  cudf::strings_column_view const& strings,
  hashed_vocabulary const& vocabulary_table,
  uint32_t max_sequence_length,
  uint32_t stride,
  bool do_lower_case,
  bool do_truncate,
  rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource());
  // end of group
}  // namespace nvtext