Skip to content

Commit 71f973a

Browse files
nikhiltrinikhiltri
committed
Limit GitHub uploads of data dumps to 1,000 files per folder
1 parent ec15298 commit 71f973a

File tree

2 files changed

+163
-1
lines changed

2 files changed

+163
-1
lines changed

‎app/Console/Commands/Dump/DumpUpload.php‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,23 @@ public function handle()
4242
$this->shell->passthru('git -C %s remote add origin %s', $repoPath, $repoRemote);
4343

4444
// Copy dumps of whitelisted tables and endpoints into the repo
45-
$this->shell->passthru('rsync -r --exclude \'.gitignore\' %s/ %s', $srcPath, $repoPath);
45+
$this->shell->passthru('rsync -r %s/ %s', $srcPath . '/getting-started', $repoPath . '/getting-started');
46+
$this->shell->passthru("for file in %s/*; do
47+
if [ -f \$file ]; then
48+
cp %s/$(basename \$file) %s/
49+
fi
50+
done
51+
", $srcPath, $srcPath, $repoPath);
52+
$this->shell->passthru('mkdir %s', $repoPath . '/json');
53+
$this->shell->passthru("for dir in %s/json/*; do
54+
if [ -d \$dir ]; then
55+
mkdir %s/json/$(basename \$dir)
56+
for file in $(ls -p \$dir | grep -v / | head -1000); do
57+
cp %s/json/$(basename \$dir)/\$file %s/json/$(basename \$dir)
58+
done
59+
fi
60+
done
61+
", $srcPath, $repoPath, $srcPath, $repoPath);
4662

4763
// Add VERSION file with current commit
4864
$this->shell->passthru('git -C %s rev-parse HEAD > %s', base_path(), $repoPath . '/VERSION');

‎app/Scopes/UnlistedScope.php‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
<?php
2+
3+
namespace App\Scopes;
4+
5+
use Carbon\Carbon;
6+
7+
use Illuminate\Database\Eloquent\Scope;
8+
use Illuminate\Database\Eloquent\Model;
9+
use Illuminate\Database\Eloquent\Builder;
10+
use Illuminate\Support\Facades\Schema;
11+
12+
class ListedScope implements Scope
13+
{
14+
/**
15+
* Apply the scope to a given Eloquent query builder.
16+
*
17+
* @param \Illuminate\Database\Eloquent\Builder $builder
18+
* @param \Illuminate\Database\Eloquent\Model $model
19+
* @return void
20+
*/
21+
public function apply(Builder $builder, Model $model)
22+
{
23+
$columns = collect(Schema::getColumnListing($model->getTable()));
24+
$builder
25+
->when($columns->contains('is_published'), function ($query) {
26+
return $query->where('is_published', '=', true);
27+
})
28+
// Logic borrows from area17/twill->/src/Models/Model->scopeVisible
29+
->when($columns->contains('publish_start_date'), function ($query) {
30+
return $query->where(function ($query2) {
31+
return $query2->where('publish_start_date', '<=', Carbon::now())
32+
->orWhereNull('publish_start_date');
33+
});
34+
})
35+
->when($columns->contains('publish_end_date'), function ($query) {
36+
return $query->where(function ($query2) {
37+
$query2->where('publish_end_date', '>=', Carbon::now())
38+
->orWhereNull('publish_end_date');
39+
});
40+
})
41+
// Account of other field names
42+
->when($columns->contains('is_private'), function ($query) {
43+
return $query->where('is_private', '=', false);
44+
})
45+
->when($columns->contains('active'), function ($query) {
46+
return $query->where('active', '=', true);
47+
});
48+
}
49+
50+
51+
public static function forSearch()
52+
{
53+
return [
54+
[
55+
'bool' => [
56+
'should' => [
57+
[
58+
'bool' => [
59+
'should' => [
60+
[
61+
'bool' => [
62+
'must_not' => [
63+
'exists' => [
64+
'field' => 'published',
65+
],
66+
],
67+
],
68+
],
69+
[
70+
'term' => [
71+
'published' => true,
72+
],
73+
],
74+
],
75+
],
76+
],
77+
[
78+
'bool' => [
79+
'should' => [
80+
[
81+
'bool' => [
82+
'must_not' => [
83+
'exists' => [
84+
'field' => 'is_published',
85+
],
86+
],
87+
],
88+
],
89+
[
90+
'term' => [
91+
'is_published' => true,
92+
],
93+
],
94+
],
95+
],
96+
],
97+
],
98+
],
99+
],
100+
[
101+
'bool' => [
102+
'should' => [
103+
[
104+
'bool' => [
105+
'must_not' => [
106+
'exists' => [
107+
'field' => 'publish_start_date',
108+
],
109+
],
110+
],
111+
],
112+
[
113+
'range' => [
114+
'publish_start_date' => [
115+
'lte' => 'now',
116+
],
117+
],
118+
],
119+
],
120+
],
121+
],
122+
[
123+
'bool' => [
124+
'should' => [
125+
[
126+
'bool' => [
127+
'must_not' => [
128+
'exists' => [
129+
'field' => 'publish_end_date',
130+
],
131+
],
132+
],
133+
],
134+
[
135+
'range' => [
136+
'publish_end_date' => [
137+
'gte' => 'now',
138+
],
139+
],
140+
],
141+
],
142+
],
143+
],
144+
];
145+
}
146+
}

0 commit comments

Comments
 (0)