Skip to content

Commit 29283f9

Browse files
committed
Add example avoid_create_0byte_files
1 parent 5a4aac0 commit 29283f9

File tree

1 file changed

+55
-0
lines changed

1 file changed

+55
-0
lines changed
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
hdfs_example: &hdfs_example
2+
config_files:
3+
- /etc/hadoop/conf/core-site.xml
4+
- /etc/hadoop/conf/hdfs-site.xml
5+
config:
6+
fs.defaultFS: 'hdfs://hadoop-nn1:8020'
7+
fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
8+
fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
9+
10+
local_fs_example: &local_fs_example
11+
config:
12+
fs.defaultFS: 'file:///'
13+
fs.hdfs.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
14+
fs.file.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
15+
io.compression.codecs: 'org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec'
16+
17+
exec:
18+
min_output_tasks: 10
19+
20+
in:
21+
type: file
22+
path_prefix: example/data
23+
parser:
24+
charset: UTF-8
25+
newline: CRLF
26+
type: csv
27+
delimiter: ','
28+
quote: '"'
29+
header_line: true
30+
stop_on_invalid_record: true
31+
columns:
32+
- {name: id, type: long}
33+
- {name: account, type: long}
34+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
35+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
36+
- {name: comment, type: string}
37+
38+
39+
out:
40+
type: hdfs
41+
<<: *local_fs_example
42+
path_prefix: /tmp/embulk-output-hdfs_example/file_
43+
file_ext: csv
44+
delete_in_advance: FILE_ONLY
45+
formatter:
46+
type: csv
47+
newline: CRLF
48+
newline_in_field: LF
49+
header_line: false
50+
charset: UTF-8
51+
quote_policy: NONE
52+
quote: '"'
53+
escape: '\'
54+
null_string: ''
55+
default_timezone: UTC

0 commit comments

Comments
 (0)