Skip to content
This repository was archived by the owner on Sep 27, 2019. It is now read-only.

Commit 6992d2b

Browse files
pmenonpmenon
committed
Propagate file options through optimization
1 parent 1f2e21e commit 6992d2b

File tree

7 files changed

+95
-23
lines changed

7 files changed

+95
-23
lines changed

‎src/include/optimizer/operators.h

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ class LogicalGet : public OperatorNode<LogicalGet> {
7373
class LogicalExternalFileGet : public OperatorNode<LogicalExternalFileGet> {
7474
public:
7575
static Operator make(oid_t get_id, ExternalFileFormat format,
76-
std::string file_name);
76+
std::string file_name, char delimiter, char quote,
77+
char escape);
7778

7879
bool operator==(const BaseOperatorNode &r) override;
7980

@@ -83,6 +84,9 @@ class LogicalExternalFileGet : public OperatorNode<LogicalExternalFileGet> {
8384
oid_t get_id;
8485
ExternalFileFormat format;
8586
std::string file_name;
87+
char delimiter;
88+
char quote;
89+
char escape;
8690
};
8791

8892
//===--------------------------------------------------------------------===//
@@ -330,14 +334,18 @@ class LogicalUpdate : public OperatorNode<LogicalUpdate> {
330334
class LogicalExportExternalFile
331335
: public OperatorNode<LogicalExportExternalFile> {
332336
public:
333-
static Operator make(ExternalFileFormat format, std::string file_name);
337+
static Operator make(ExternalFileFormat format, std::string file_name,
338+
char delimiter, char quote, char escape);
334339

335340
bool operator==(const BaseOperatorNode &r) override;
336341

337342
hash_t Hash() const override;
338343

339344
ExternalFileFormat format;
340345
std::string file_name;
346+
char delimiter;
347+
char quote;
348+
char escape;
341349
};
342350

343351
//===--------------------------------------------------------------------===//
@@ -410,7 +418,8 @@ class PhysicalIndexScan : public OperatorNode<PhysicalIndexScan> {
410418
class ExternalFileScan : public OperatorNode<ExternalFileScan> {
411419
public:
412420
static Operator make(oid_t get_id, ExternalFileFormat format,
413-
std::string file_name);
421+
std::string file_name, char delimiter, char quote,
422+
char escape);
414423

415424
bool operator==(const BaseOperatorNode &r) override;
416425

@@ -420,6 +429,9 @@ class ExternalFileScan : public OperatorNode<ExternalFileScan> {
420429
oid_t get_id;
421430
ExternalFileFormat format;
422431
std::string file_name;
432+
char delimiter;
433+
char quote;
434+
char escape;
423435
};
424436

425437
//===--------------------------------------------------------------------===//
@@ -617,14 +629,18 @@ class PhysicalUpdate : public OperatorNode<PhysicalUpdate> {
617629
class PhysicalExportExternalFile
618630
: public OperatorNode<PhysicalExportExternalFile> {
619631
public:
620-
static Operator make(ExternalFileFormat format, std::string file_name);
632+
static Operator make(ExternalFileFormat format, std::string file_name,
633+
char delimiter, char quote, char escape);
621634

622635
bool operator==(const BaseOperatorNode &r) override;
623636

624637
hash_t Hash() const override;
625638

626639
ExternalFileFormat format;
627640
std::string file_name;
641+
char delimiter;
642+
char quote;
643+
char escape;
628644
};
629645

630646
//===--------------------------------------------------------------------===//

‎src/include/parser/copy_statement.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,13 @@ class CopyStatement : public SQLStatement {
6262
std::string file_path;
6363

6464
// The format of the file
65-
ExternalFileFormat format;
65+
ExternalFileFormat format = ExternalFileFormat::CSV;
6666

6767
bool is_from;
6868

69-
char delimiter;
69+
char delimiter = ',';
70+
char quote = '"';
71+
char escape = '"';
7072
};
7173

7274
} // namespace parser

‎src/optimizer/operators.cpp

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -68,19 +68,24 @@ bool LogicalGet::operator==(const BaseOperatorNode &r) {
6868
//===--------------------------------------------------------------------===//
6969

7070
Operator LogicalExternalFileGet::make(oid_t get_id, ExternalFileFormat format,
71-
std::string file_name) {
71+
std::string file_name, char delimiter,
72+
char quote, char escape) {
7273
auto *get = new LogicalExternalFileGet();
7374
get->get_id = get_id;
7475
get->format = format;
7576
get->file_name = std::move(file_name);
77+
get->delimiter = delimiter;
78+
get->quote = quote;
79+
get->escape = escape;
7680
return Operator(get);
7781
}
7882

7983
bool LogicalExternalFileGet::operator==(const BaseOperatorNode &node) {
8084
if (node.GetType() != OpType::LogicalExternalFileGet) return false;
8185
const auto &get = *static_cast<const LogicalExternalFileGet *>(&node);
8286
return (get_id == get.get_id && format == get.format &&
83-
file_name == get.file_name);
87+
file_name == get.file_name && delimiter == get.delimiter &&
88+
quote == get.quote && escape == get.escape);
8489
}
8590

8691
hash_t LogicalExternalFileGet::Hash() const {
@@ -89,6 +94,9 @@ hash_t LogicalExternalFileGet::Hash() const {
8994
hash = HashUtil::CombineHashes(hash, HashUtil::Hash(&format));
9095
hash = HashUtil::CombineHashes(
9196
hash, HashUtil::HashBytes(file_name.data(), file_name.length()));
97+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&delimiter, 1));
98+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&quote, 1));
99+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&escape, 1));
92100
return hash;
93101
}
94102

@@ -446,25 +454,34 @@ Operator LogicalLimit::make(int64_t offset, int64_t limit) {
446454
// External file output
447455
//===--------------------------------------------------------------------===//
448456
Operator LogicalExportExternalFile::make(ExternalFileFormat format,
449-
std::string file_name) {
457+
std::string file_name, char delimiter,
458+
char quote, char escape) {
450459
auto *export_op = new LogicalExportExternalFile();
451460
export_op->format = format;
452461
export_op->file_name = std::move(file_name);
462+
export_op->delimiter = delimiter;
463+
export_op->quote = quote;
464+
export_op->escape = escape;
453465
return Operator(export_op);
454466
}
455467

456468
bool LogicalExportExternalFile::operator==(const BaseOperatorNode &node) {
457469
if (node.GetType() != OpType::LogicalExportExternalFile) return false;
458470
const auto &export_op =
459471
*static_cast<const LogicalExportExternalFile *>(&node);
460-
return (format == export_op.format && file_name == export_op.file_name);
472+
return (format == export_op.format && file_name == export_op.file_name &&
473+
delimiter == export_op.delimiter && quote == export_op.quote &&
474+
escape == export_op.escape);
461475
}
462476

463477
hash_t LogicalExportExternalFile::Hash() const {
464478
hash_t hash = BaseOperatorNode::Hash();
465479
hash = HashUtil::CombineHashes(hash, HashUtil::Hash(&format));
466480
hash = HashUtil::CombineHashes(
467481
hash, HashUtil::HashBytes(file_name.data(), file_name.length()));
482+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&delimiter, 1));
483+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&quote, 1));
484+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&escape, 1));
468485
return hash;
469486
}
470487

@@ -567,19 +584,24 @@ hash_t PhysicalIndexScan::Hash() const {
567584
// Physical external file scan
568585
//===--------------------------------------------------------------------===//
569586
Operator ExternalFileScan::make(oid_t get_id, ExternalFileFormat format,
570-
std::string file_name) {
587+
std::string file_name, char delimiter,
588+
char quote, char escape) {
571589
auto *get = new ExternalFileScan();
572590
get->get_id = get_id;
573591
get->format = format;
574592
get->file_name = file_name;
593+
get->delimiter = delimiter;
594+
get->quote = quote;
595+
get->escape = escape;
575596
return Operator(get);
576597
}
577598

578599
bool ExternalFileScan::operator==(const BaseOperatorNode &node) {
579600
if (node.GetType() != OpType::QueryDerivedScan) return false;
580601
const auto &get = *static_cast<const ExternalFileScan *>(&node);
581602
return (get_id == get.get_id && format == get.format &&
582-
file_name == get.file_name);
603+
file_name == get.file_name && delimiter == get.delimiter &&
604+
quote == get.quote && escape == get.escape);
583605
}
584606

585607
hash_t ExternalFileScan::Hash() const {
@@ -588,6 +610,9 @@ hash_t ExternalFileScan::Hash() const {
588610
hash = HashUtil::CombineHashes(hash, HashUtil::Hash(&format));
589611
hash = HashUtil::CombineHashes(
590612
hash, HashUtil::HashBytes(file_name.data(), file_name.length()));
613+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&delimiter, 1));
614+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&quote, 1));
615+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&escape, 1));
591616
return hash;
592617
}
593618

@@ -845,25 +870,34 @@ Operator PhysicalUpdate::make(
845870
// PhysicalExportExternalFile
846871
//===--------------------------------------------------------------------===//
847872
Operator PhysicalExportExternalFile::make(ExternalFileFormat format,
848-
std::string file_name) {
873+
std::string file_name, char delimiter,
874+
char quote, char escape) {
849875
auto *export_op = new PhysicalExportExternalFile();
850876
export_op->format = format;
851877
export_op->file_name = file_name;
878+
export_op->delimiter = delimiter;
879+
export_op->quote = quote;
880+
export_op->escape = escape;
852881
return Operator(export_op);
853882
}
854883

855884
bool PhysicalExportExternalFile::operator==(const BaseOperatorNode &node) {
856885
if (node.GetType() != OpType::ExportExternalFile) return false;
857886
const auto &export_op =
858887
*static_cast<const PhysicalExportExternalFile *>(&node);
859-
return (format == export_op.format && file_name == export_op.file_name);
888+
return (format == export_op.format && file_name == export_op.file_name &&
889+
delimiter == export_op.delimiter && quote == export_op.quote &&
890+
escape == export_op.escape);
860891
}
861892

862893
hash_t PhysicalExportExternalFile::Hash() const {
863894
hash_t hash = BaseOperatorNode::Hash();
864895
hash = HashUtil::CombineHashes(hash, HashUtil::Hash(&format));
865896
hash = HashUtil::CombineHashes(
866897
hash, HashUtil::HashBytes(file_name.data(), file_name.length()));
898+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&delimiter, 1));
899+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&quote, 1));
900+
hash = HashUtil::CombineHashes(hash, HashUtil::HashBytes(&escape, 1));
867901
return hash;
868902
}
869903

‎src/optimizer/plan_generator.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,8 @@ void PlanGenerator::Visit(const ExternalFileScan *op) {
119119

120120
// Create the plan
121121
output_plan_.reset(
122-
new planner::CSVScanPlan(op->file_name, std::move(cols)));
122+
new planner::CSVScanPlan(op->file_name, std::move(cols),
123+
op->delimiter, op->quote, op->escape));
123124
break;
124125
}
125126
}
@@ -364,7 +365,8 @@ void PlanGenerator::Visit(const PhysicalUpdate *op) {
364365

365366
void PlanGenerator::Visit(const PhysicalExportExternalFile *op) {
366367
unique_ptr<planner::AbstractPlan> export_plan{
367-
new planner::ExportExternalFilePlan(op->file_name)};
368+
new planner::ExportExternalFilePlan(op->file_name, op->delimiter,
369+
op->quote, op->escape)};
368370
export_plan->AddChild(move(children_plans_[0]));
369371
output_plan_ = move(export_plan);
370372
}

‎src/optimizer/query_to_operator_transformer.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,8 @@ void QueryToOperatorTransformer::Visit(parser::CopyStatement *op) {
367367

368368
auto get_op =
369369
std::make_shared<OperatorExpression>(LogicalExternalFileGet::make(
370-
GetAndIncreaseGetId(), op->format, op->file_path));
370+
GetAndIncreaseGetId(), op->format, op->file_path, op->delimiter,
371+
op->quote, op->escape));
371372

372373
auto target_table =
373374
catalog::Catalog::GetInstance()
@@ -386,8 +387,9 @@ void QueryToOperatorTransformer::Visit(parser::CopyStatement *op) {
386387
} else {
387388
op->table->Accept(this);
388389
}
389-
auto export_op = std::make_shared<OperatorExpression>(
390-
LogicalExportExternalFile::make(op->format, op->file_path));
390+
auto export_op =
391+
std::make_shared<OperatorExpression>(LogicalExportExternalFile::make(
392+
op->format, op->file_path, op->delimiter, op->quote, op->escape));
391393
export_op->PushChild(output_expr_);
392394
output_expr_ = export_op;
393395
}

‎src/optimizer/rule_impls.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,8 @@ void LogicalExternalFileGetToPhysical::Transform(
455455
const auto *get = input->Op().As<LogicalExternalFileGet>();
456456

457457
auto result_plan = std::make_shared<OperatorExpression>(
458-
ExternalFileScan::make(get->get_id, get->format, get->file_name));
458+
ExternalFileScan::make(get->get_id, get->format, get->file_name,
459+
get->delimiter, get->quote, get->escape));
459460

460461
PELOTON_ASSERT(input->Children().empty());
461462

@@ -837,11 +838,12 @@ void LogicalExportToPhysicalExport::Transform(
837838
std::shared_ptr<OperatorExpression> input,
838839
std::vector<std::shared_ptr<OperatorExpression>> &transformed,
839840
UNUSED_ATTRIBUTE OptimizeContext *context) const {
840-
const auto *logical_export = input->Op().As<LogicalExportExternalFile>();
841+
const auto *export_op = input->Op().As<LogicalExportExternalFile>();
841842

842843
auto result_plan =
843844
std::make_shared<OperatorExpression>(PhysicalExportExternalFile::make(
844-
logical_export->format, logical_export->file_name));
845+
export_op->format, export_op->file_name, export_op->delimiter,
846+
export_op->quote, export_op->escape));
845847

846848
std::vector<std::shared_ptr<OperatorExpression>> children = input->Children();
847849
PELOTON_ASSERT(children.size() == 1);

‎src/parser/postgresparser.cpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1508,6 +1508,8 @@ parser::PrepareStatement *PostgresParser::PrepareTransform(PrepareStmt *root) {
15081508
parser::CopyStatement *PostgresParser::CopyTransform(CopyStmt *root) {
15091509
static constexpr char kDelimiterTok[] = "delimiter";
15101510
static constexpr char kFormatTok[] = "format";
1511+
static constexpr char kQuoteTok[] = "quote";
1512+
static constexpr char kEscapeTok[] = "escape";
15111513

15121514
// The main return value
15131515
auto *result = new CopyStatement();
@@ -1538,12 +1540,24 @@ parser::CopyStatement *PostgresParser::CopyTransform(CopyStmt *root) {
15381540
auto *format_val = reinterpret_cast<value *>(def_elem->arg);
15391541
result->format = StringToExternalFileFormat(format_val->val.str);
15401542
}
1543+
1544+
// Check quote
1545+
if (strncmp(def_elem->defname, kQuoteTok, sizeof(kQuoteTok)) == 0) {
1546+
auto *quote_val = reinterpret_cast<value *>(def_elem->arg);
1547+
result->quote = *quote_val->val.str;
1548+
}
1549+
1550+
// Check escape
1551+
if (strncmp(def_elem->defname, kEscapeTok, sizeof(kEscapeTok)) == 0) {
1552+
auto *escape_val = reinterpret_cast<value *>(def_elem->arg);
1553+
result->escape = *escape_val->val.str;
1554+
}
15411555
}
15421556

15431557
return result;
15441558
}
15451559

1546-
// Analyze statment is parsed with vacuum statment.
1560+
// Analyze statment is parsed with vacuum statement.
15471561
parser::AnalyzeStatement *PostgresParser::VacuumTransform(VacuumStmt *root) {
15481562
if (root->options != VACOPT_ANALYZE) {
15491563
throw NotImplementedException("Vacuum not supported.");

0 commit comments

Comments
 (0)