Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

Commit 99b8c8f

Browse files
imeoerimeoer
committed
add basical load local support
support source image from -l and STDIN fix load error issue fix build issue fix build issue fix diff panic issue for timeout testing fix up to date issue add debug progress update to hijack post add close for after post add header to postRaw fix progress issue fix tar issue improve progress show update desc from doc update by comments fix err redefine issue add err check for hijack fix quiest upload issue close conn if copy error update code structure update by comments add prefix for hijack err
1 parent b99be5b commit 99b8c8f

File tree

6 files changed

+389
-23
lines changed

6 files changed

+389
-23
lines changed

‎api/client/load.go

Lines changed: 287 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,317 @@
11
package client
22

33
import (
4+
"encoding/json"
45
"errors"
6+
"fmt"
57
"io"
8+
"io/ioutil"
9+
"os"
10+
"path/filepath"
11+
"strings"
612

7-
"golang.org/x/net/context"
8-
13+
"github.com/docker/engine-api/types"
914
Cli "github.com/hyperhq/hypercli/cli"
15+
"github.com/hyperhq/hypercli/image"
16+
"github.com/hyperhq/hypercli/pkg/archive"
1017
"github.com/hyperhq/hypercli/pkg/jsonmessage"
1118
flag "github.com/hyperhq/hypercli/pkg/mflag"
19+
"github.com/hyperhq/hypercli/pkg/progress"
20+
"github.com/hyperhq/hypercli/pkg/streamformatter"
21+
"github.com/hyperhq/hypercli/pkg/symlink"
22+
"golang.org/x/net/context"
1223
)
1324

14-
// CmdLoad loads an image from a tar archive.
25+
const (
26+
unsupported = "Unsupported image version"
27+
)
28+
29+
type manifestItem struct {
30+
Config string
31+
RepoTags []string
32+
Layers []string
33+
}
34+
35+
type readCloser struct {
36+
io.Reader
37+
NeedClose io.ReadCloser
38+
}
39+
40+
func (rc readCloser) Close() error {
41+
return rc.NeedClose.Close()
42+
}
43+
44+
func safePath(base, path string) (string, error) {
45+
return symlink.FollowSymlinkInScope(filepath.Join(base, path), base)
46+
}
47+
48+
func removeExistLayers(tmpDir string, existLayers []string, layerPaths []string) error {
49+
keepLayerPaths := make(map[string]bool)
50+
for _, _layerPath := range layerPaths[len(existLayers):] {
51+
layerPath := filepath.Join(tmpDir, _layerPath)
52+
info, err := os.Lstat(layerPath)
53+
if err != nil {
54+
return err
55+
}
56+
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
57+
if _realPath, err := filepath.EvalSymlinks(layerPath); err == nil {
58+
realPath := filepath.Join(filepath.Base(filepath.Dir(_realPath)), "layer.tar")
59+
keepLayerPaths[realPath] = true
60+
}
61+
}
62+
}
63+
for idx := range existLayers {
64+
layerPath := layerPaths[idx]
65+
if _, ok := keepLayerPaths[layerPath]; !ok {
66+
if err := os.Remove(filepath.Join(tmpDir, layerPath)); err != nil {
67+
continue
68+
}
69+
}
70+
}
71+
return nil
72+
}
73+
74+
func (cli *DockerCli) getExistLayers(ctx context.Context, tmpDir string) ([]string, []string, error) {
75+
manifestPath, err := safePath(tmpDir, "manifest.json")
76+
if err != nil {
77+
return nil, nil, err
78+
}
79+
80+
manifestFile, err := os.Open(manifestPath)
81+
if err != nil {
82+
return nil, nil, err
83+
}
84+
defer manifestFile.Close()
85+
86+
var manifest []manifestItem
87+
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
88+
return nil, nil, err
89+
}
90+
91+
allLayers := make([][]string, 0)
92+
repoTags := make([][]string, 0)
93+
layerPaths := make([]string, 0)
94+
for _, m := range manifest {
95+
configPath, err := safePath(tmpDir, m.Config)
96+
if err != nil {
97+
return nil, nil, err
98+
}
99+
config, err := ioutil.ReadFile(configPath)
100+
if err != nil {
101+
return nil, nil, err
102+
}
103+
img, err := image.NewFromJSON(config)
104+
if err != nil {
105+
return nil, nil, err
106+
}
107+
108+
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
109+
return nil, nil, errors.New(unsupported)
110+
}
111+
112+
layerPaths = append(layerPaths, m.Layers...)
113+
114+
layers := make([]string, 0)
115+
116+
for _, diffID := range img.RootFS.DiffIDs {
117+
layers = append(layers, string(diffID))
118+
}
119+
120+
allLayers = append(allLayers, layers)
121+
repoTags = append(repoTags, m.RepoTags)
122+
}
123+
diffRet, err := cli.client.ImageDiff(ctx, allLayers, repoTags)
124+
if err != nil {
125+
return nil, nil, err
126+
}
127+
return diffRet.ExistLayers, layerPaths, nil
128+
}
129+
130+
func (cli *DockerCli) ImageLoadFromTar(ctx context.Context, tr io.Reader, quiet bool) (*types.ImageLoadResponse, error) {
131+
tmpDir, err := ioutil.TempDir("", "hyper-pull-local-")
132+
if err != nil {
133+
return nil, err
134+
}
135+
defer os.RemoveAll(tmpDir)
136+
137+
if err := archive.Untar(tr, tmpDir, &archive.TarOptions{NoLchown: true}); err != nil {
138+
return nil, err
139+
}
140+
141+
if !quiet {
142+
fmt.Fprintln(cli.out, "Diffing local image with remote image...")
143+
}
144+
145+
existLayers, layerPaths, err := cli.getExistLayers(ctx, tmpDir)
146+
if err != nil {
147+
return nil, err
148+
}
149+
150+
if err := removeExistLayers(tmpDir, existLayers, layerPaths); err != nil {
151+
return nil, err
152+
}
153+
154+
fs, err := archive.Tar(tmpDir, archive.Gzip)
155+
if err != nil {
156+
return nil, err
157+
}
158+
defer fs.Close()
159+
160+
hasNewLayers := len(existLayers) != len(layerPaths)
161+
if hasNewLayers {
162+
if !quiet {
163+
fmt.Fprintln(cli.out, "Preparing to upload image...")
164+
}
165+
}
166+
167+
tarTmpDir, err := ioutil.TempDir("", "hyper-pull-local-")
168+
if err != nil {
169+
return nil, err
170+
}
171+
defer os.RemoveAll(tarTmpDir)
172+
173+
tarPath := filepath.Join(tarTmpDir, "image.tar")
174+
175+
tf, err := os.Create(tarPath)
176+
if err != nil {
177+
return nil, err
178+
}
179+
defer tf.Close()
180+
181+
_, err = io.Copy(tf, fs)
182+
if err != nil {
183+
return nil, err
184+
}
185+
os.RemoveAll(tmpDir)
186+
187+
info, err := tf.Stat()
188+
if err != nil {
189+
return nil, err
190+
}
191+
tf, err = os.Open(tarPath)
192+
if err != nil {
193+
return nil, err
194+
}
195+
196+
resp, err := cli.client.ImageLoadLocal(ctx, quiet, info.Size())
197+
if err != nil {
198+
return nil, err
199+
}
200+
201+
if !hasNewLayers || quiet {
202+
go func() {
203+
_, err := io.Copy(resp.Conn, tf)
204+
if err != nil {
205+
fmt.Fprintln(cli.out, err.Error())
206+
resp.Conn.Close()
207+
return
208+
}
209+
tf.Close()
210+
}()
211+
return &types.ImageLoadResponse{
212+
Body: resp.Conn,
213+
JSON: true,
214+
}, nil
215+
}
216+
217+
pr, pw := io.Pipe()
218+
progressOutput := streamformatter.NewJSONStreamFormatter().NewProgressOutput(pw, false)
219+
progressReader := progress.NewProgressReader(tf, progressOutput, info.Size(), "", "Uploading image")
220+
221+
go func() {
222+
_, err := io.Copy(resp.Conn, progressReader)
223+
if err != nil {
224+
fmt.Fprintln(cli.out, err.Error())
225+
resp.Conn.Close()
226+
return
227+
}
228+
pw.CloseWithError(io.EOF)
229+
}()
230+
231+
return &types.ImageLoadResponse{
232+
Body: readCloser{io.MultiReader(pr, resp.Conn), resp.Conn},
233+
JSON: true,
234+
}, nil
235+
}
236+
237+
// ImageDiff diff an image layers with local and imaged
238+
func (cli *DockerCli) ImageLoadFromDaemon(ctx context.Context, name string, quiet bool) (*types.ImageLoadResponse, error) {
239+
if !quiet {
240+
fmt.Fprintln(cli.out, "Loading image from local docker daemon...")
241+
}
242+
243+
tr, err := cli.client.ImageSaveTarFromDaemon(ctx, []string{name})
244+
if err != nil {
245+
return nil, err
246+
}
247+
defer tr.Close()
248+
249+
return cli.ImageLoadFromTar(ctx, tr, quiet)
250+
}
251+
252+
// CmdLoad load a local image or a tar file
15253
//
16254
// The tar archive is read from STDIN by default, or from a tar archive file.
17255
//
18256
// Usage: docker load [OPTIONS]
19257
func (cli *DockerCli) CmdLoad(args ...string) error {
20-
cmd := Cli.Subcmd("load", nil, Cli.DockerCommands["load"].Description, true)
21-
infile := cmd.String([]string{"i", "-input"}, "", "Read from a remote archive file compressed with gzip, bzip, or xz")
258+
cmd := Cli.Subcmd("load", nil, "Load a local image or a tar file", true)
259+
local := cmd.String([]string{"l", "-local"}, "", "Read from a local image")
260+
infile := cmd.String([]string{"i", "-input"}, "", "Read from a local or remote archive file compressed with gzip, bzip, or xz, instead of STDIN")
22261
quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Do not show load process")
23262
cmd.Require(flag.Exact, 0)
24263
cmd.ParseFlags(args, true)
25264

26-
if *infile == "" {
27-
return errors.New("remote archive must be specified via --input")
265+
*infile = strings.TrimSpace(*infile)
266+
*local = strings.TrimSpace(*local)
267+
268+
var stdin io.Reader = cli.in
269+
270+
if *infile == "" && *local == "" && stdin == nil {
271+
return errors.New("source image must be specified via --input, --local or STDIN")
28272
}
29273

30-
var input struct {
31-
FromSrc string `json:"fromSrc"`
32-
Quiet bool `json:"quiet"`
274+
var response *types.ImageLoadResponse
275+
var err error
276+
277+
if *local != "" {
278+
// Load from local docker daemon
279+
response, err = cli.ImageLoadFromDaemon(context.Background(), *local, *quiet)
280+
} else if *infile != "" {
281+
if strings.HasPrefix(*infile, "http://") ||
282+
strings.HasPrefix(*infile, "https://") ||
283+
strings.HasPrefix(*infile, "ftp://") {
284+
var input struct {
285+
FromSrc string `json:"fromSrc"`
286+
Quiet bool `json:"quiet"`
287+
}
288+
input.FromSrc = *infile
289+
input.Quiet = *quiet
290+
// Load from remote URL
291+
response, err = cli.client.ImageLoad(context.Background(), input)
292+
} else {
293+
// Load from local tar
294+
var af *os.File
295+
af, err = os.Open(*infile)
296+
if err != nil {
297+
return err
298+
}
299+
defer af.Close()
300+
response, err = cli.ImageLoadFromTar(context.Background(), af, *quiet)
301+
}
302+
} else if stdin != nil {
303+
// Load from STDIN
304+
response, err = cli.ImageLoadFromTar(context.Background(), stdin, *quiet)
33305
}
34-
input.FromSrc = *infile
35-
input.Quiet = *quiet
36306

37-
response, err := cli.client.ImageLoad(context.Background(), input)
38307
if err != nil {
39308
return err
40309
}
310+
311+
if response == nil {
312+
return nil
313+
}
314+
41315
defer response.Body.Close()
42316

43317
if response.JSON {

‎vendor/src/github.com/docker/engine-api/client/hijack.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,11 @@ func (cli *Client) postHijacked(ctx context.Context, path string, query url.Valu
6969
defer clientconn.Close()
7070

7171
// Server hijacks the connection, error 'connection closed' expected
72-
_, err = clientconn.Do(req)
72+
resp, err := clientconn.Do(req)
7373

7474
rwc, br := clientconn.Hijack()
7575

76-
return types.HijackedResponse{Conn: rwc, Reader: br}, err
76+
return types.HijackedResponse{Conn: rwc, Reader: br, Resp: resp}, err
7777
}
7878

7979
func tlsDial(network, addr string, config *tls.Config) (net.Conn, error) {

‎vendor/src/github.com/docker/engine-api/client/image_load.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,19 @@ package client
33
import (
44
"net/url"
55

6-
"golang.org/x/net/context"
7-
86
"github.com/docker/engine-api/types"
7+
"golang.org/x/net/context"
98
)
109

1110
// ImageLoad loads an image in the docker host from the client host.
1211
// It's up to the caller to close the io.ReadCloser returned by
1312
// this function.
14-
func (cli *Client) ImageLoad(ctx context.Context, input interface{}) (types.ImageLoadResponse, error) {
13+
func (cli *Client) ImageLoad(ctx context.Context, input interface{}) (*types.ImageLoadResponse, error) {
1514
resp, err := cli.post(ctx, "/images/load", url.Values{}, input, nil)
1615
if err != nil {
17-
return types.ImageLoadResponse{}, err
16+
return nil, err
1817
}
19-
return types.ImageLoadResponse{
18+
return &types.ImageLoadResponse{
2019
Body: resp.body,
2120
JSON: resp.header.Get("Content-Type") == "application/json",
2221
}, nil

0 commit comments

Comments
 (0)