Refactor get_col_names()

Use parse() and move to utility header

Author: vincentlaucsb

include/internal/csv_reader.cpp

@@ -5,48 +5,6 @@
 #include "csv_reader.hpp"
 
 namespace csv {
-    namespace internals {
-        CSV_INLINE std::string format_row(const std::vector<std::string>& row, csv::string_view delim) {
-            /** Print a CSV row */
-            std::stringstream ret;
-            for (size_t i = 0; i < row.size(); i++) {
-                ret << row[i];
-                if (i + 1 < row.size()) ret << delim;
-                else ret << '\n';
-            }
-            ret.flush();
-
-            return ret.str();
-        }
-
-        /** Return the selected header row from a parsed head buffer. */
-        CSV_INLINE std::vector<std::string> _get_col_names(csv::string_view head, CSVFormat format) {
-            // Parse the CSV
-            auto trim_chars = format.get_trim_chars();
-            std::stringstream source(head.data());
-            RowCollection rows;
-
-            StreamParser<std::stringstream> parser(source, format);
-            parser.set_output(rows);
-            parser.next();
-
-            return CSVRow(std::move(rows[format.get_header()]));
-        }
-    }
-
-    /** Return a CSV's column names. */
-    CSV_INLINE std::vector<std::string> get_col_names(csv::string_view filename, CSVFormat format) {
-        auto head = internals::get_csv_head(filename);
-
-        /** Guess delimiter and header row */
-        if (format.guess_delim()) {
-            auto guess_result = guess_format(filename, format.get_possible_delims());
-            format.delimiter(guess_result.delim).header_row(guess_result.header_row);
-        }
-
-        return internals::_get_col_names(head, format);
-    }
-
     /** Reads an arbitrarily large CSV file using memory-mapped IO.
      *
      *  **Details:** Reads the first block of a CSV file synchronously to get information
@@ -104,11 +62,8 @@ namespace csv {
 
     /** Return the CSV's column names as a vector of strings. */
     CSV_INLINE std::vector<std::string> CSVReader::get_col_names() const {
-        if (this->col_names) {
-            return this->col_names->get_col_names();
-        }
-
-        return std::vector<std::string>();
+        return (this->col_names) ? this->col_names->get_col_names() : 
+            std::vector<std::string>();
     }
 
     /** Return the index of the column name if found or
@@ -263,9 +218,9 @@ namespace csv {
 
                     if (policy == VariableColumnPolicy::THROW) {
                         if (errored_row.size() < this->n_cols)
-                            throw std::runtime_error("Line too short " + internals::format_row(errored_row));
+                            throw std::runtime_error("Line too short " + std::string(errored_row.raw_str()));
 
-                        throw std::runtime_error("Line too long " + internals::format_row(errored_row));
+                        throw std::runtime_error("Line too long " + std::string(errored_row.raw_str()));
                     }
 
                     continue;

include/internal/csv_reader.hpp

@@ -27,17 +27,6 @@
 
 /** The all encompassing namespace */
 namespace csv {
-    /** Stuff that is generally not of interest to end-users */
-    namespace internals {
-        std::string format_row(const std::vector<std::string>& row, csv::string_view delim = ", ");
-
-        std::vector<std::string> _get_col_names( csv::string_view head, const CSVFormat format = CSVFormat::guess_csv());
-    }
-
-    std::vector<std::string> get_col_names(
-        csv::string_view filename,
-        const CSVFormat format = CSVFormat::guess_csv());
-
 #if CSV_ENABLE_THREADS
     inline void join_worker(std::thread& worker) {
         if (worker.joinable()) worker.join();
@@ -386,11 +375,16 @@ namespace csv {
         template<typename TStream,
             csv::enable_if_t<std::is_base_of<std::istream, TStream>::value, int> = 0>
         void init_from_stream(TStream& source, CSVFormat format) {
+            // C4316: StreamParser may have over-aligned SIMD members; heap allocation
+            // alignment is handled correctly at runtime via the allocator on supported
+            // platforms. Suppress the MSVC false-positive here.
+            CSV_MSVC_PUSH_DISABLE(4316)
             this->init_parser(
                 std::unique_ptr<internals::IBasicCSVParser>(
                     new internals::StreamParser<TStream>(source, format, this->col_names)
                 )
             );
+            CSV_MSVC_POP
         }
 
         /** Read initial chunk to get metadata */

include/internal/csv_row.cpp

@@ -63,6 +63,17 @@ namespace csv {
         return ret;
     }
 
+    CSV_INLINE csv::string_view CSVRow::raw_str() const noexcept {
+        if (!data) return csv::string_view();
+        const csv::string_view full = data->data;
+        if (data_start >= full.size()) return csv::string_view();
+        const size_t end = full.find('\n', data_start);
+        const size_t len = (end == csv::string_view::npos)
+            ? (full.size() - data_start)
+            : (end - data_start);
+        return full.substr(data_start, len);
+    }
+
     /** Build a map from column names to values for a given row. */
     CSV_INLINE std::unordered_map<std::string, std::string> CSVRow::to_unordered_map() const {
         std::unordered_map<std::string, std::string> row_map;

include/internal/csv_row.hpp

@@ -363,6 +363,15 @@ namespace csv {
          *       it materializes all fields as owning strings.
          */
         operator std::vector<std::string>() const;
+
+        /** Return a string_view of the raw bytes of this row as they appear in
+         *  the underlying parse buffer, up to (but not including) the trailing
+         *  newline character.
+         *
+         *  @warning The view is only valid for as long as the CSVRow (and its
+         *           associated data chunk) remains alive.
+         */
+        csv::string_view raw_str() const noexcept;
         ///@}
 
         /** A random access iterator over the contents of a CSV row.

include/internal/csv_stat.cpp

@@ -153,7 +153,7 @@ namespace csv {
                 }
             }
             else if (this->reader.get_format().get_variable_column_policy() == VariableColumnPolicy::THROW) {
-                throw std::runtime_error("Line has different length than the others " + internals::format_row(*current_record));
+                throw std::runtime_error("Line has different length than the others " + std::string(current_record->raw_str()));
             }
 
             ++current_record;

include/internal/csv_utility.hpp

@@ -34,7 +34,7 @@ namespace csv {
      *  @par Example
      *  @snippet tests/test_read_csv.cpp Parse Example
      */
-    inline CSVReader parse(csv::string_view in, CSVFormat format = CSVFormat::guess_csv()) {
+    inline CSVReader parse(csv::string_view in, const CSVFormat& format = CSVFormat::guess_csv()) {
         std::unique_ptr<std::istream> ss(new std::stringstream(std::string(in)));
         return CSVReader(std::move(ss), format);
     }
@@ -105,11 +105,20 @@ namespace csv {
         };
     }
 
+    /** Get the column names of a CSV file using just the first 500KB. */
+    inline std::vector<std::string> get_col_names(
+        csv::string_view filename,
+        const CSVFormat& format = CSVFormat::guess_csv()) {
+        auto head = internals::get_csv_head(filename);
+        return parse_unsafe(head, format).get_col_names();
+    }
+
     /** Find the position of a column in a CSV file or CSV_NOT_FOUND otherwise. */
-    inline int get_col_pos(csv::string_view filename, csv::string_view col_name,
+    inline long long get_col_pos(csv::string_view filename, csv::string_view col_name,
         const CSVFormat& format = CSVFormat::guess_csv()) {
-        CSVReader reader(filename, format);
-        return reader.index_of(col_name);
+        auto col_names = get_col_names(filename, format);
+        return col_names.empty() ? CSV_NOT_FOUND :
+            std::distance(col_names.begin(), std::find(col_names.begin(), col_names.end(), col_name));
     }
     ///@}
 }

programs/csv_info.cpp

@@ -12,8 +12,12 @@ int main(int argc, char** argv) {
     std::string file = argv[1];
     auto info = get_file_info(file);
 
-    std::cout << file << std::endl
-        << "Columns: " << internals::format_row(info.col_names, ", ")
+    std::cout << file << std::endl << "Columns: ";
+    for (size_t i = 0; i < info.col_names.size(); i++) {
+        if (i) std::cout << ", ";
+        std::cout << info.col_names[i];
+    }
+    std::cout << std::endl
         << "Dimensions: " << info.n_rows << " rows x " << info.n_cols << " columns" << std::endl
         << "Delimiter: " << info.delim << std::endl;
 

single_include_test/file1.cpp

@@ -12,8 +12,12 @@ int foobar(int argc, char** argv) {
     std::string file = argv[1];
     auto info = get_file_info(file);
 
-    std::cout << file << std::endl
-        << "Columns: " << internals::format_row(info.col_names, ", ")
+    std::cout << file << std::endl << "Columns: ";
+    for (size_t i = 0; i < info.col_names.size(); i++) {
+        if (i) std::cout << ", ";
+        std::cout << info.col_names[i];
+    }
+    std::cout << std::endl
         << "Dimensions: " << info.n_rows << " rows x " << info.n_cols << " columns" << std::endl
         << "Delimiter: " << info.delim << std::endl;
 

tests/test_read_csv_file.cpp

@@ -15,7 +15,7 @@ using std::string;
 
 #ifndef __EMSCRIPTEN__
 TEST_CASE("col_pos() Test", "[test_col_pos]") {
-    int pos = get_col_pos(
+    auto pos = get_col_pos(
         "./tests/data/real_data/2015_StateDepartment.csv",
         "Entity Type");
     REQUIRE(pos == 1);