dfe_io_dsv.hpp

Go to the documentation of this file. Or view the newest version in sPHENIX GitHub for file dfe_io_dsv.hpp

// SPDX-License-Identifier: MIT
// Copyright 2015,2018-2020 Moritz Kiehn
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.


#pragma once

#include <algorithm>
#include <array>
#include <fstream>
#include <limits>
#include <sstream>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#include <utility>
#include <vector>

namespace dfe {
namespace io_dsv_impl {

template<char Delimiter>
class DsvWriter {
public:
  DsvWriter() = delete;
  DsvWriter(const DsvWriter&) = delete;
  DsvWriter(DsvWriter&&) = default;
  ~DsvWriter() = default;
  DsvWriter& operator=(const DsvWriter&) = delete;
  DsvWriter& operator=(DsvWriter&&) = default;

  DsvWriter(
    const std::vector<std::string>& columns, const std::string& path,
    int precision = std::numeric_limits<double>::max_digits10);

  template<typename Arg0, typename... Args>
  void append(Arg0&& arg0, Args&&... args);

private:
  std::ofstream m_file;
  std::size_t m_num_columns;

  // enable_if to prevent this overload to be used for std::vector<T> as well
  template<typename T>
  static std::enable_if_t<
    std::is_arithmetic<std::decay_t<T>>::value
      or std::is_convertible<T, std::string>::value,
    unsigned>
  write(T&& x, std::ostream& os);
  template<typename T, typename Allocator>
  static unsigned write(const std::vector<T, Allocator>& xs, std::ostream& os);
};

template<char Delimiter>
class DsvReader {
public:
  DsvReader() = delete;
  DsvReader(const DsvReader&) = delete;
  DsvReader(DsvReader&&) = default;
  ~DsvReader() = default;
  DsvReader& operator=(const DsvReader&) = delete;
  DsvReader& operator=(DsvReader&&) = default;

  DsvReader(const std::string& path);

  bool read(std::vector<std::string>& columns);

  std::size_t num_lines() const { return m_num_lines; }

private:
  std::ifstream m_file;
  std::string m_line;
  std::size_t m_num_lines = 0;
};

template<char Delimiter, typename NamedTuple>
class NamedTupleDsvWriter {
public:
  NamedTupleDsvWriter() = delete;
  NamedTupleDsvWriter(const NamedTupleDsvWriter&) = delete;
  NamedTupleDsvWriter(NamedTupleDsvWriter&&) = default;
  ~NamedTupleDsvWriter() = default;
  NamedTupleDsvWriter& operator=(const NamedTupleDsvWriter&) = delete;
  NamedTupleDsvWriter& operator=(NamedTupleDsvWriter&&) = default;

  NamedTupleDsvWriter(
    const std::string& path,
    int precision = std::numeric_limits<double>::max_digits10)
    : m_writer(colum_names(), path, precision) {}

  void append(const NamedTuple& record) {
    append_impl(
      record, std::make_index_sequence<
                std::tuple_size<typename NamedTuple::Tuple>::value>{});
  }

private:
  DsvWriter<Delimiter> m_writer;

  static std::vector<std::string> colum_names() {
    const auto& from_record = NamedTuple::names();
    return {from_record.begin(), from_record.end()};
  }
  template<std::size_t... I>
  void append_impl(const NamedTuple& values, std::index_sequence<I...>) {
    using std::get;
    m_writer.append(get<I>(values)...);
  }
};

// string conversion helper functions

template<typename T>
static void
parse(const std::string& str, T& value) {
  // TODO use somthing w/ lower overhead then stringstream e.g. std::from_chars
  std::istringstream is(str);
  is >> value;
}

template<char Delimiter, typename NamedTuple>
class NamedTupleDsvReader {
public:
  NamedTupleDsvReader() = delete;
  NamedTupleDsvReader(const NamedTupleDsvReader&) = delete;
  NamedTupleDsvReader(NamedTupleDsvReader&&) = default;
  ~NamedTupleDsvReader() = default;
  NamedTupleDsvReader& operator=(const NamedTupleDsvReader&) = delete;
  NamedTupleDsvReader& operator=(NamedTupleDsvReader&&) = default;

  NamedTupleDsvReader(
    const std::string& path,
    const std::vector<std::string>& optional_columns = {},
    bool verify_header = true);

  bool read(NamedTuple& record);

  template<typename T>
  bool read(NamedTuple& record, std::vector<T>& extra);

  std::size_t num_extra_columns() const { return m_extra_columns.size(); }
  std::size_t num_records() const { return m_reader.num_lines() - 1u; }

private:
  // the equivalent std::tuple-like type
  using Tuple = typename NamedTuple::Tuple;

  DsvReader<Delimiter> m_reader;
  std::vector<std::string> m_columns;
  // #columns is fixed to a reasonable value after reading the header
  std::size_t m_num_columns = SIZE_MAX;
  // map tuple index to column index in the file, SIZE_MAX for missing elements
  std::array<std::size_t, std::tuple_size<Tuple>::value> m_tuple_column_map;
  // column indices that do not map to a tuple items
  std::vector<std::size_t> m_extra_columns;

  void use_default_columns();
  void parse_header(const std::vector<std::string>& optional_columns);
  template<std::size_t... I>
  void parse_record(NamedTuple& record, std::index_sequence<I...>) const {
    // see namedtuple_impl::print_tuple for explanation
    // allow different column ordering on file and optional columns
    using Vacuum = int[];
    (void)Vacuum{(parse_element<I>(record), 0)...};
  }
  template<std::size_t I>
  void parse_element(NamedTuple& record) const {
    using std::get;
    if (m_tuple_column_map[I] != SIZE_MAX) {
      parse(m_columns[m_tuple_column_map[I]], get<I>(record));
    }
  }
};

// implementation writer

template<char Delimiter>
inline DsvWriter<Delimiter>::DsvWriter(
  const std::vector<std::string>& columns, const std::string& path,
  int precision)
  : m_file(
    path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc)
  , m_num_columns(columns.size()) {
  if (not m_file.is_open() or m_file.fail()) {
    throw std::runtime_error("Could not open file '" + path + "'");
  }
  m_file.precision(precision);
  if (m_num_columns == 0) {
    throw std::invalid_argument("No columns were specified");
  }
  // write column names as header row
  append(columns);
}

template<char Delimiter>
template<typename Arg0, typename... Args>
inline void
DsvWriter<Delimiter>::append(Arg0&& arg0, Args&&... args) {
  // we can only check how many columns were written after they have been
  // written. write to temporary first to prevent bad data on file.
  std::stringstream line;
  // ensure consistent formatting
  line.precision(m_file.precision());
  unsigned written_columns[] = {
    // write the first item without a delimiter and store columns written
    write(std::forward<Arg0>(arg0), line),
    // for all other items, write the delimiter followed by the item itself
    // (<expr1>, <expr2>) use the comma operator (yep, ',' in c++ is a weird
    // but helpful operator) to execute both expression and return the return
    // value of the last one, i.e. here thats the number of columns written.
    // the ... pack expansion creates this expression for all arguments
    (line << Delimiter, write(std::forward<Args>(args), line))...,
  };
  line << '\n';
  // validate that the total number of written columns matches the specs.
  unsigned total_columns = 0;
  for (auto nc : written_columns) {
    total_columns += nc;
  }
  if (total_columns < m_num_columns) {
    throw std::invalid_argument("Not enough columns");
  }
  if (m_num_columns < total_columns) {
    throw std::invalid_argument("Too many columns");
  }
  // write the line to disk and check that it actually happened
  m_file << line.rdbuf();
  if (not m_file.good()) {
    throw std::runtime_error("Could not write data to file");
  }
}

template<char Delimiter>
template<typename T>
inline std::enable_if_t<
  std::is_arithmetic<std::decay_t<T>>::value
    or std::is_convertible<T, std::string>::value,
  unsigned>
DsvWriter<Delimiter>::write(T&& x, std::ostream& os) {
  os << x;
  return 1u;
}

template<char Delimiter>
template<typename T, typename Allocator>
inline unsigned
DsvWriter<Delimiter>::write(
  const std::vector<T, Allocator>& xs, std::ostream& os) {
  unsigned n = 0;
  for (const auto& x : xs) {
    if (0 < n) {
      os << Delimiter;
    }
    os << x;
    n += 1;
  }
  return n;
}

// implementation reader

template<char Delimiter>
inline DsvReader<Delimiter>::DsvReader(const std::string& path)
  : m_file(path, std::ios_base::binary | std::ios_base::in) {
  if (not m_file.is_open() or m_file.fail()) {
    throw std::runtime_error("Could not open file '" + path + "'");
  }
}

template<char Delimiter>
inline bool
DsvReader<Delimiter>::read(std::vector<std::string>& columns) {
  // read the next line and check for both end-of-file and errors
  std::getline(m_file, m_line);
  if (m_file.eof()) {
    return false;
  }
  if (m_file.fail()) {
    throw std::runtime_error(
      "Could not read line " + std::to_string(m_num_lines));
  }
  m_num_lines += 1;

  // split the line into columns
  columns.clear();
  for (std::string::size_type pos = 0; pos < m_line.size();) {
    auto del = m_line.find_first_of(Delimiter, pos);
    if (del == std::string::npos) {
      // reached the end of the line; also determines the last column
      columns.emplace_back(m_line, pos);
      break;
    } else {
      columns.emplace_back(m_line, pos, del - pos);
      // start next column search after the delimiter
      pos = del + 1;
    }
  }
  return true;
}

// implementation named tuple reader

template<char Delimiter, typename NamedTuple>
inline NamedTupleDsvReader<Delimiter, NamedTuple>::NamedTupleDsvReader(
  const std::string& path, const std::vector<std::string>& optional_columns,
  bool verify_header)
  : m_reader(path) {
  // optional columns only work if we verify the header
  if ((not optional_columns.empty()) and (not verify_header)) {
    throw std::runtime_error(
      "Optional columns can not be used without header verification");
  }
  // first line is always the header
  if (not m_reader.read(m_columns)) {
    throw std::runtime_error("Could not read header from '" + path + "'");
  }
  if (verify_header) {
    parse_header(optional_columns);
  } else {
    use_default_columns();
  }
}

template<char Delimiter, typename NamedTuple>
inline bool
NamedTupleDsvReader<Delimiter, NamedTuple>::read(NamedTuple& record) {
  if (not m_reader.read(m_columns)) {
    return false;
  }
  // check for consistent entries per-line
  if (m_columns.size() < m_num_columns) {
    throw std::runtime_error(
      "Too few columns in line " + std::to_string(m_reader.num_lines()));
  }
  if (m_num_columns < m_columns.size()) {
    throw std::runtime_error(
      "Too many columns in line " + std::to_string(m_reader.num_lines()));
  }
  // convert to tuple
  parse_record(
    record, std::make_index_sequence<std::tuple_size<Tuple>::value>{});
  return true;
}

template<char Delimiter, typename NamedTuple>
template<typename T>
inline bool
NamedTupleDsvReader<Delimiter, NamedTuple>::read(
  NamedTuple& record, std::vector<T>& extra) {
  // parse columns belonging to the regular record
  if (not read(record)) {
    return false;
  }
  // parse extra columns
  extra.resize(m_extra_columns.size());
  for (std::size_t i = 0; i < m_extra_columns.size(); ++i) {
    parse(m_columns[m_extra_columns[i]], extra[i]);
  }
  return true;
}

template<char Delimiter, typename NamedTuple>
inline void
NamedTupleDsvReader<Delimiter, NamedTuple>::use_default_columns() {
  // assume row content is identical in content and order to the tuple
  m_num_columns = std::tuple_size<Tuple>::value;
  for (std::size_t i = 0; i < m_tuple_column_map.size(); ++i) {
    m_tuple_column_map[i] = i;
  }
  // no extra columns by construction
  m_extra_columns.clear();
}

template<char Delimiter, typename NamedTuple>
inline void
NamedTupleDsvReader<Delimiter, NamedTuple>::parse_header(
  const std::vector<std::string>& optional_columns) {
  const auto& names = NamedTuple::names();

  // the number of header columns fixes the expected number of data columns
  m_num_columns = m_columns.size();

  // check that all non-optional columns are available
  for (const auto& name : names) {
    // no need to for availability if the column is optional
    auto o = std::find(optional_columns.begin(), optional_columns.end(), name);
    if (o != optional_columns.end()) {
      continue;
    }
    // missing, non-optional column mean we can not continue
    auto c = std::find(m_columns.begin(), m_columns.end(), name);
    if (c == m_columns.end()) {
      throw std::runtime_error("Missing header column '" + name + "'");
    }
  }

  // ensure missing columns are correctly marked as such
  m_tuple_column_map.fill(SIZE_MAX);

  // determine column-tuple mapping and extra column indices
  m_extra_columns.clear();
  for (std::size_t i = 0; i < m_columns.size(); ++i) {
    // find the position of the column in the tuple.
    auto it = std::find(names.begin(), names.end(), m_columns[i]);
    if (it != names.end()) {
      // establish mapping between column and tuple item position
      m_tuple_column_map[std::distance(names.begin(), it)] = i;
    } else {
      // record non-tuple columns
      m_extra_columns.push_back(i);
    }
  }
}

} // namespace io_dsv_impl

using CsvWriter = io_dsv_impl::DsvWriter<','>;

using TsvWriter = io_dsv_impl::DsvWriter<'\t'>;

template<typename T>
using NamedTupleCsvWriter = io_dsv_impl::NamedTupleDsvWriter<',', T>;

template<typename T>
using NamedTupleCsvReader = io_dsv_impl::NamedTupleDsvReader<',', T>;

template<typename T>
using NamedTupleTsvWriter = io_dsv_impl::NamedTupleDsvWriter<'\t', T>;

template<typename T>
using NamedTupleTsvReader = io_dsv_impl::NamedTupleDsvReader<'\t', T>;

} // namespace dfe