From 95d4a7ce04e7558c16b6cbf07bf3d63179a76f60 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Thu, 5 Mar 2020 13:43:36 -0700 Subject: [PATCH 1/7] zot: run GC after manifest removal Clients today expect the repo to clean up if there are unused blobs, not to manually delete things they think are unused. Let's do that, and use umoci's code to do it since it's tested and works. v2: also run GC on update as well as delete v3: fix up error return paths needing two args Signed-off-by: Serge Hallyn Signed-off-by: Tycho Andersen --- go.mod | 1 + go.sum | 88 +++++++++++++++++++++++++++++++++++++ pkg/storage/storage.go | 22 ++++++++++ pkg/storage/storage_test.go | 2 - 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 8ea91718..1b5f83b5 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/mitchellh/mapstructure v1.1.2 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 // indirect github.com/nmcclain/ldap v0.0.0-20191021200707-3b3b69a7e9e3 + github.com/openSUSE/umoci v0.4.5 github.com/opencontainers/distribution-spec v1.0.0-rc0 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 diff --git a/go.sum b/go.sum index b4f6d719..5753ed7f 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,15 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apex/log v1.1.1 h1:BwhRZ0qbjYtTob0I+2M+smavV0kOC8XgcnGZcyL9liA= +github.com/apex/log v1.1.1/go.mod h1:Ls949n1HFtXfbDcjiTTFQqkVUrte0puoIBfO3SVgwOA= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/aphistic/sweet v0.3.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.23.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -25,13 +33,22 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a h1:yU/FENpkHYISWsQrbr3pcZOBj0EuRjPzNc1+dTCLu44= @@ -71,9 +88,12 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= @@ -86,9 +106,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -98,11 +121,19 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.3 h1:CkLseiEYMM/fRb0RIg9mXB+Iwgmle+U9KGFu+JCO4Ec= +github.com/klauspost/compress v1.8.3/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= +github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -110,10 +141,15 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -123,18 +159,27 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 h1:D9EvfGQvlkKaDr2CRKN++7HbSXbefUNDrPq60T+g24s= github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484/go.mod h1:O1EljZ+oHprtxDDPHiMWVo/5dBT6PlvWX5PSwj80aBA= github.com/nmcclain/ldap v0.0.0-20191021200707-3b3b69a7e9e3 h1:NNis9uuNpG5h97Dvxxo53Scg02qBg+3Nfabg6zjFGu8= github.com/nmcclain/ldap v0.0.0-20191021200707-3b3b69a7e9e3/go.mod h1:YtrVB1/v9Td9SyjXpjYVmbdKgj9B0nPTBsdGUxy0i8U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openSUSE/umoci v0.4.5 h1:MZgFLy5Jl3VKe5uEOU9c25FoySbx3vUXeXLw4Jf6aRs= +github.com/openSUSE/umoci v0.4.5/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= github.com/opencontainers/distribution-spec v1.0.0-rc0 h1:xMzwhweo1gjvEo74mQjGTLau0TD3ACyTEC1310NbuSQ= github.com/opencontainers/distribution-spec v1.0.0-rc0/go.mod h1:copR2flp+jTEvQIFMb6MIx45OkrxzqyjszPDT3hx/5Q= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runtime-spec v1.0.1 h1:wY4pOY8fBdSIvs9+IDHC55thBuEulhzfSgKeC1yFvzQ= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= @@ -156,15 +201,33 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4= +github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.17.2 h1:RMRHFw2+wF7LO0QqtELQwo8hqSmqISyCJeFeAAuWcRo= github.com/rs/zerolog v1.17.2/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/smartystreets/gunit v1.0.4/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= @@ -182,6 +245,7 @@ github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -197,6 +261,10 @@ github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e/go.mod h1:eycb github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= github.com/swaggo/swag v1.6.3 h1:N+uVPGP4H2hXoss2pt5dctoSUPKKRInr6qcTMOm0usI= github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= @@ -204,6 +272,10 @@ github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljT github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/go-mtree v0.4.4 h1:+CncqETnSpxBCCUhRnNQBvxhsjWXNuc+ExZsLSNaj5o= +github.com/vbatts/go-mtree v0.4.4/go.mod h1:3sazBqLG4bZYmgRTgdh9X3iKTzwBpp5CrREJDzrNSXY= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -214,11 +286,14 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -229,13 +304,17 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc= +golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -243,8 +322,12 @@ golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7 h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg= +golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -258,6 +341,8 @@ golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74 h1:4cFkmztxtMslUX2SctSl+blCyXfpzhGOy9LhKAqSMA4= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190913181337-0240832f5c3d h1:JG5eBjADEFhCRMAdfzvIYKuT/2E9oOzCDJMqMxD5LVw= +golang.org/x/tools v0.0.0-20190913181337-0240832f5c3d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -267,12 +352,15 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 2ee742ac..eaf9637a 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,6 +1,7 @@ package storage import ( + "context" "crypto/sha256" "encoding/json" "fmt" @@ -16,6 +17,7 @@ import ( guuid "github.com/gofrs/uuid" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/openSUSE/umoci" "github.com/rs/zerolog" ) @@ -435,6 +437,16 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType return "", err } + oci, err := umoci.OpenLayout(dir) + if err != nil { + return "", err + } + defer oci.Close() + + if err := oci.GC(context.Background()); err != nil { + return "", err + } + return desc.Digest.String(), nil } @@ -502,6 +514,16 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { return err } + oci, err := umoci.OpenLayout(dir) + if err != nil { + return err + } + defer oci.Close() + + if err := oci.GC(context.Background()); err != nil { + return err + } + p := path.Join(dir, "blobs") p = path.Join(p, digest.Algorithm().String()) p = path.Join(p, digest.Encoded()) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 71497e48..0419104f 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -135,8 +135,6 @@ func TestAPIs(t *testing.T) { So(err, ShouldNotBeNil) }) - err = il.DeleteBlob("test", d.String()) - So(err, ShouldBeNil) }) err = il.DeleteBlobUpload("test", v) From 94afc30661744ecf24e49b25984e3b00208cb471 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Thu, 5 Mar 2020 17:46:19 -0700 Subject: [PATCH 2/7] routes: lock everything that modifies blobs Now that we're GCing blobs on delete/update manifest, we should lock the blob queries so that they don't race with each other. This is a pretty coarse grained lock, there's probably a better way to do this. Signed-off-by: Tycho Andersen --- pkg/api/routes.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 25b23588..02e2722c 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -20,6 +20,7 @@ import ( "sort" "strconv" "strings" + "sync" _ "github.com/anuvu/zot/docs" // nolint (golint) - as required by swaggo "github.com/anuvu/zot/errors" @@ -40,16 +41,25 @@ const ( ) type RouteHandler struct { - c *Controller + c *Controller + blobLock sync.Mutex } func NewRouteHandler(c *Controller) *RouteHandler { - rh := &RouteHandler{c: c} + rh := &RouteHandler{c: c, blobLock: sync.Mutex{}} rh.SetupRoutes() return rh } +func (rh *RouteHandler) blobLockWrapper(f func (w http.ResponseWriter, r *http.Request)) func (w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + rh.blobLock.Lock() + f(w, r) + rh.blobLock.Unlock() + } +} + func (rh *RouteHandler) SetupRoutes() { rh.c.Router.Use(AuthHandler(rh.c)) g := rh.c.Router.PathPrefix(RoutePrefix).Subrouter() @@ -61,25 +71,25 @@ func (rh *RouteHandler) SetupRoutes() { g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), rh.GetManifest).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.UpdateManifest).Methods("PUT") + rh.blobLockWrapper(rh.UpdateManifest)).Methods("PUT") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.DeleteManifest).Methods("DELETE") + rh.blobLockWrapper(rh.DeleteManifest)).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), rh.CheckBlob).Methods("HEAD") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), rh.GetBlob).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), - rh.DeleteBlob).Methods("DELETE") + rh.blobLockWrapper(rh.DeleteBlob)).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()), - rh.CreateBlobUpload).Methods("POST") + rh.blobLockWrapper(rh.CreateBlobUpload)).Methods("POST") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), rh.GetBlobUpload).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.PatchBlobUpload).Methods("PATCH") + rh.blobLockWrapper(rh.PatchBlobUpload)).Methods("PATCH") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.UpdateBlobUpload).Methods("PUT") + rh.blobLockWrapper(rh.UpdateBlobUpload)).Methods("PUT") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.DeleteBlobUpload).Methods("DELETE") + rh.blobLockWrapper(rh.DeleteBlobUpload)).Methods("DELETE") g.HandleFunc("/_catalog", rh.ListRepositories).Methods("GET") g.HandleFunc("/", From 404c83fbc125dff6ce457c86ed3046b1226f1552 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 6 Mar 2020 12:23:20 -0800 Subject: [PATCH 3/7] bazel: fix bazel build failures --- WORKSPACE | 327 +++++++++++++++++++++++++++++++++++----- pkg/api/BUILD.bazel | 5 +- pkg/cli/BUILD.bazel | 2 - pkg/storage/BUILD.bazel | 1 + 4 files changed, 297 insertions(+), 38 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 40ba6f80..0c6232d9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -326,8 +326,8 @@ go_repository( go_repository( name = "com_github_golang_protobuf", importpath = "github.com/golang/protobuf", - sum = "h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=", - version = "v1.3.1", + sum = "h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=", + version = "v1.3.2", ) go_repository( @@ -403,8 +403,8 @@ go_repository( go_repository( name = "com_github_json_iterator_go", importpath = "github.com/json-iterator/go", - sum = "h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=", - version = "v1.1.8", + sum = "h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=", + version = "v1.1.9", ) go_repository( @@ -438,8 +438,8 @@ go_repository( go_repository( name = "com_github_konsorten_go_windows_terminal_sequences", importpath = "github.com/konsorten/go-windows-terminal-sequences", - sum = "h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=", - version = "v1.0.1", + sum = "h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=", + version = "v1.0.2", ) go_repository( @@ -459,8 +459,8 @@ go_repository( go_repository( name = "com_github_kr_pty", importpath = "github.com/kr/pty", - sum = "h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=", - version = "v1.1.1", + sum = "h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=", + version = "v1.1.8", ) go_repository( @@ -487,8 +487,8 @@ go_repository( go_repository( name = "com_github_mattn_go_isatty", importpath = "github.com/mattn/go-isatty", - sum = "h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=", - version = "v0.0.8", + sum = "h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=", + version = "v0.0.9", ) go_repository( @@ -578,8 +578,8 @@ go_repository( go_repository( name = "com_github_pkg_errors", importpath = "github.com/pkg/errors", - sum = "h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=", - version = "v0.8.1", + sum = "h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=", + version = "v0.9.1", ) go_repository( @@ -641,8 +641,8 @@ go_repository( go_repository( name = "com_github_rogpeppe_fastuuid", importpath = "github.com/rogpeppe/fastuuid", - sum = "h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng=", - version = "v0.0.0-20150106093220-6724a57986af", + sum = "h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=", + version = "v1.2.0", ) go_repository( @@ -669,15 +669,15 @@ go_repository( go_repository( name = "com_github_sirupsen_logrus", importpath = "github.com/sirupsen/logrus", - sum = "h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=", - version = "v1.2.0", + sum = "h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=", + version = "v1.4.2", ) go_repository( name = "com_github_smartystreets_assertions", importpath = "github.com/smartystreets/assertions", - sum = "h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=", - version = "v0.0.0-20180927180507-b2de0cb4f26d", + sum = "h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=", + version = "v1.0.1", ) go_repository( @@ -746,15 +746,15 @@ go_repository( go_repository( name = "com_github_stretchr_objx", importpath = "github.com/stretchr/objx", - sum = "h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=", - version = "v0.1.1", + sum = "h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=", + version = "v0.2.0", ) go_repository( name = "com_github_stretchr_testify", importpath = "github.com/stretchr/testify", - sum = "h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=", - version = "v1.3.0", + sum = "h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=", + version = "v1.4.0", ) go_repository( @@ -795,8 +795,8 @@ go_repository( go_repository( name = "com_github_urfave_cli", importpath = "github.com/urfave/cli", - sum = "h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=", - version = "v1.20.0", + sum = "h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=", + version = "v1.22.1", ) go_repository( @@ -837,8 +837,8 @@ go_repository( go_repository( name = "in_gopkg_check_v1", importpath = "gopkg.in/check.v1", - sum = "h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=", - version = "v1.0.0-20180628173108-788fd7840127", + sum = "h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=", + version = "v1.0.0-20190902080502-41f04d3bba15", ) go_repository( @@ -914,8 +914,8 @@ go_repository( go_repository( name = "org_golang_x_net", importpath = "golang.org/x/net", - sum = "h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=", - version = "v0.0.0-20190620200207-3b0461eec859", + sum = "h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc=", + version = "v0.0.0-20190912160710-24e19bdeb0f2", ) go_repository( @@ -928,15 +928,15 @@ go_repository( go_repository( name = "org_golang_x_sync", importpath = "golang.org/x/sync", - sum = "h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=", - version = "v0.0.0-20190423024810-112230192c58", + sum = "h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=", + version = "v0.0.0-20190911185100-cd5d95a43a6e", ) go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4=", - version = "v0.0.0-20190610200419-93c9922d18ae", + sum = "h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg=", + version = "v0.0.0-20190913121621-c3b328c6e5a7", ) go_repository( @@ -956,8 +956,8 @@ go_repository( go_repository( name = "org_golang_x_tools", importpath = "golang.org/x/tools", - sum = "h1:4cFkmztxtMslUX2SctSl+blCyXfpzhGOy9LhKAqSMA4=", - version = "v0.0.0-20190828213141-aed303cbaa74", + sum = "h1:JG5eBjADEFhCRMAdfzvIYKuT/2E9oOzCDJMqMxD5LVw=", + version = "v0.0.0-20190913181337-0240832f5c3d", ) go_repository( @@ -1197,3 +1197,262 @@ go_repository( sum = "h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=", version = "v1.51.0", ) + +go_repository( + name = "com_github_apex_log", + importpath = "github.com/apex/log", + sum = "h1:BwhRZ0qbjYtTob0I+2M+smavV0kOC8XgcnGZcyL9liA=", + version = "v1.1.1", +) + +go_repository( + name = "com_github_aphistic_golf", + importpath = "github.com/aphistic/golf", + sum = "h1:2KLQMJ8msqoPHIPDufkxVcoTtcmE5+1sL9950m4R9Pk=", + version = "v0.0.0-20180712155816-02c07f170c5a", +) + +go_repository( + name = "com_github_aphistic_sweet", + importpath = "github.com/aphistic/sweet", + sum = "h1:xZTMfCoMsjWubPNxOBODluBC4qfGP0CdRJ88jon46XE=", + version = "v0.3.0", +) + +go_repository( + name = "com_github_aws_aws_sdk_go", + importpath = "github.com/aws/aws-sdk-go", + sum = "h1:eVJT2C99cAjZlBY8+CJovf6AwrSANzAcYNuxdCB+SPk=", + version = "v1.23.21", +) + +go_repository( + name = "com_github_aybabtme_rgbterm", + importpath = "github.com/aybabtme/rgbterm", + sum = "h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo=", + version = "v0.0.0-20170906152045-cc83f3b3ce59", +) + +go_repository( + name = "com_github_cpuguy83_go_md2man_v2", + importpath = "github.com/cpuguy83/go-md2man/v2", + sum = "h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=", + version = "v2.0.0-20190314233015-f79a8a8ca69d", +) + +go_repository( + name = "com_github_creack_pty", + importpath = "github.com/creack/pty", + sum = "h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A=", + version = "v1.1.7", +) + +go_repository( + name = "com_github_cyphar_filepath_securejoin", + importpath = "github.com/cyphar/filepath-securejoin", + sum = "h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=", + version = "v0.2.2", +) + +go_repository( + name = "com_github_docker_go_units", + importpath = "github.com/docker/go-units", + sum = "h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=", + version = "v0.4.0", +) + +go_repository( + name = "com_github_fatih_color", + importpath = "github.com/fatih/color", + sum = "h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=", + version = "v1.7.0", +) + +go_repository( + name = "com_github_google_uuid", + importpath = "github.com/google/uuid", + sum = "h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=", + version = "v1.1.1", +) + +go_repository( + name = "com_github_hpcloud_tail", + importpath = "github.com/hpcloud/tail", + sum = "h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=", + version = "v1.0.0", +) + +go_repository( + name = "com_github_jmespath_go_jmespath", + importpath = "github.com/jmespath/go-jmespath", + sum = "h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=", + version = "v0.0.0-20180206201540-c2b33e8439af", +) + +go_repository( + name = "com_github_jpillora_backoff", + importpath = "github.com/jpillora/backoff", + sum = "h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME=", + version = "v0.0.0-20180909062703-3050d21c67d7", +) + +go_repository( + name = "com_github_klauspost_compress", + importpath = "github.com/klauspost/compress", + sum = "h1:CkLseiEYMM/fRb0RIg9mXB+Iwgmle+U9KGFu+JCO4Ec=", + version = "v1.8.3", +) + +go_repository( + name = "com_github_klauspost_cpuid", + importpath = "github.com/klauspost/cpuid", + sum = "h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=", + version = "v1.2.1", +) + +go_repository( + name = "com_github_klauspost_pgzip", + importpath = "github.com/klauspost/pgzip", + sum = "h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM=", + version = "v1.2.1", +) + +go_repository( + name = "com_github_mattn_go_colorable", + importpath = "github.com/mattn/go-colorable", + sum = "h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=", + version = "v0.1.2", +) + +go_repository( + name = "com_github_mgutz_ansi", + importpath = "github.com/mgutz/ansi", + sum = "h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=", + version = "v0.0.0-20170206155736-9520e82c474b", +) + +go_repository( + name = "com_github_mohae_deepcopy", + importpath = "github.com/mohae/deepcopy", + sum = "h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=", + version = "v0.0.0-20170929034955-c48cc78d4826", +) + +go_repository( + name = "com_github_onsi_ginkgo", + importpath = "github.com/onsi/ginkgo", + sum = "h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=", + version = "v1.10.1", +) + +go_repository( + name = "com_github_onsi_gomega", + importpath = "github.com/onsi/gomega", + sum = "h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=", + version = "v1.7.0", +) + +go_repository( + name = "com_github_opencontainers_runtime_spec", + importpath = "github.com/opencontainers/runtime-spec", + sum = "h1:wY4pOY8fBdSIvs9+IDHC55thBuEulhzfSgKeC1yFvzQ=", + version = "v1.0.1", +) + +go_repository( + name = "com_github_opensuse_umoci", + importpath = "github.com/openSUSE/umoci", + sum = "h1:MZgFLy5Jl3VKe5uEOU9c25FoySbx3vUXeXLw4Jf6aRs=", + version = "v0.4.5", +) + +go_repository( + name = "com_github_rootless_containers_proto", + importpath = "github.com/rootless-containers/proto", + sum = "h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4=", + version = "v0.1.0", +) + +go_repository( + name = "com_github_russross_blackfriday_v2", + importpath = "github.com/russross/blackfriday/v2", + sum = "h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=", + version = "v2.0.1", +) + +go_repository( + name = "com_github_sergi_go_diff", + importpath = "github.com/sergi/go-diff", + sum = "h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=", + version = "v1.0.0", +) + +go_repository( + name = "com_github_shurcool_sanitized_anchor_name", + importpath = "github.com/shurcooL/sanitized_anchor_name", + sum = "h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=", + version = "v1.0.0", +) + +go_repository( + name = "com_github_smartystreets_go_aws_auth", + importpath = "github.com/smartystreets/go-aws-auth", + sum = "h1:hp2CYQUINdZMHdvTdXtPOY2ainKl4IoMcpAXEf2xj3Q=", + version = "v0.0.0-20180515143844-0c1422d1fdb9", +) + +go_repository( + name = "com_github_smartystreets_gunit", + importpath = "github.com/smartystreets/gunit", + sum = "h1:tpTjnuH7MLlqhoD21vRoMZbMIi5GmBsAJDFyF67GhZA=", + version = "v1.0.4", +) + +go_repository( + name = "com_github_tj_assert", + importpath = "github.com/tj/assert", + sum = "h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA=", + version = "v0.0.0-20171129193455-018094318fb0", +) + +go_repository( + name = "com_github_tj_go_elastic", + importpath = "github.com/tj/go-elastic", + sum = "h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc=", + version = "v0.0.0-20171221160941-36157cbbebc2", +) + +go_repository( + name = "com_github_tj_go_kinesis", + importpath = "github.com/tj/go-kinesis", + sum = "h1:m74UWYy+HBs+jMFR9mdZU6shPewugMyH5+GV6LNgW8w=", + version = "v0.0.0-20171128231115-08b17f58cb1b", +) + +go_repository( + name = "com_github_tj_go_spin", + importpath = "github.com/tj/go-spin", + sum = "h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=", + version = "v1.1.0", +) + +go_repository( + name = "com_github_vbatts_go_mtree", + importpath = "github.com/vbatts/go-mtree", + sum = "h1:+CncqETnSpxBCCUhRnNQBvxhsjWXNuc+ExZsLSNaj5o=", + version = "v0.4.4", +) + +go_repository( + name = "in_gopkg_fsnotify_v1", + importpath = "gopkg.in/fsnotify.v1", + sum = "h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=", + version = "v1.4.7", +) + +go_repository( + name = "in_gopkg_tomb_v1", + importpath = "gopkg.in/tomb.v1", + sum = "h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=", + version = "v1.0.0-20141024135613-dd632973f1e7", +) diff --git a/pkg/api/BUILD.bazel b/pkg/api/BUILD.bazel index 92d09ea6..e1b5b2ed 100644 --- a/pkg/api/BUILD.bazel +++ b/pkg/api/BUILD.bazel @@ -24,9 +24,7 @@ go_library( "@com_github_gorilla_handlers//:go_default_library", "@com_github_gorilla_mux//:go_default_library", "@com_github_json_iterator_go//:go_default_library", - "@com_github_mitchellh_mapstructure//:go_default_library", "@com_github_opencontainers_distribution_spec//:go_default_library", - "@com_github_opencontainers_go_digest//:go_default_library", "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", "@com_github_swaggo_http_swagger//:go_default_library", "@org_golang_x_crypto//bcrypt:go_default_library", @@ -43,7 +41,10 @@ go_test( embed = [":go_default_library"], race = "on", deps = [ + "@com_github_chartmuseum_auth//:go_default_library", + "@com_github_mitchellh_mapstructure//:go_default_library", "@com_github_nmcclain_ldap//:go_default_library", + "@com_github_opencontainers_go_digest//:go_default_library", "@com_github_smartystreets_goconvey//convey:go_default_library", "@in_gopkg_resty_v1//:go_default_library", ], diff --git a/pkg/cli/BUILD.bazel b/pkg/cli/BUILD.bazel index 5d121447..4bc2b3a9 100644 --- a/pkg/cli/BUILD.bazel +++ b/pkg/cli/BUILD.bazel @@ -8,8 +8,6 @@ go_library( deps = [ "//errors:go_default_library", "//pkg/api:go_default_library", - "//pkg/compliance:go_default_library", - "//pkg/compliance/v1_0_0:go_default_library", "//pkg/storage:go_default_library", "@com_github_mitchellh_mapstructure//:go_default_library", "@com_github_opencontainers_distribution_spec//:go_default_library", diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel index a795afd7..3efaa617 100644 --- a/pkg/storage/BUILD.bazel +++ b/pkg/storage/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "@com_github_gofrs_uuid//:go_default_library", "@com_github_opencontainers_go_digest//:go_default_library", "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", + "@com_github_opensuse_umoci//:go_default_library", "@com_github_rs_zerolog//:go_default_library", ], ) From 4774aa81b38d2f843ca00b884246c6d1f6067681 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Mon, 9 Mar 2020 15:08:31 -0600 Subject: [PATCH 4/7] compliance: don't leak test directory Signed-off-by: Tycho Andersen --- pkg/compliance/v1_0_0/check_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/compliance/v1_0_0/check_test.go b/pkg/compliance/v1_0_0/check_test.go index d3f034b2..4a0ab592 100644 --- a/pkg/compliance/v1_0_0/check_test.go +++ b/pkg/compliance/v1_0_0/check_test.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io/ioutil" + "os" "testing" "time" @@ -56,7 +57,6 @@ func startServer() (*api.Controller, string) { panic(err) } - //defer os.RemoveAll(dir) ctrl.Config.Storage.RootDirectory = dir go func() { // this blocks @@ -80,4 +80,5 @@ func startServer() (*api.Controller, string) { func stopServer(ctrl *api.Controller) { ctrl.Server.Shutdown(context.Background()) + os.RemoveAll(ctrl.Config.Storage.RootDirectory) } From fe471a3c352fa4f9c8dbf008ca6ba62e6b1b4c40 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 18 Mar 2020 15:42:06 -0700 Subject: [PATCH 5/7] gc: fix test cases since umoci GC is more strict umoci GC enforces a valid index.json and current tests were a little lax. --- go.mod | 2 +- go.sum | 3 +++ pkg/compliance/v1_0_0/check.go | 45 +++++++++++++++++++++++++++++++--- pkg/storage/storage.go | 7 +++++- pkg/storage/storage_test.go | 16 +++++++++++- 5 files changed, 67 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 1b5f83b5..b4c144fb 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/mitchellh/mapstructure v1.1.2 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 // indirect github.com/nmcclain/ldap v0.0.0-20191021200707-3b3b69a7e9e3 - github.com/openSUSE/umoci v0.4.5 + github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258 github.com/opencontainers/distribution-spec v1.0.0-rc0 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 diff --git a/go.sum b/go.sum index 5753ed7f..7b7c2fb4 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -172,6 +173,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openSUSE/umoci v0.4.5 h1:MZgFLy5Jl3VKe5uEOU9c25FoySbx3vUXeXLw4Jf6aRs= github.com/openSUSE/umoci v0.4.5/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= +github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258 h1:/8Yu54FufyHHQgIZ/wLy+BLQyzk0gbOG24xf5suWOOI= +github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= github.com/opencontainers/distribution-spec v1.0.0-rc0 h1:xMzwhweo1gjvEo74mQjGTLau0TD3ACyTEC1310NbuSQ= github.com/opencontainers/distribution-spec v1.0.0-rc0/go.mod h1:copR2flp+jTEvQIFMb6MIx45OkrxzqyjszPDT3hx/5Q= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index 71a49b1f..4f0af291 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -481,7 +481,20 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.StatusCode(), ShouldEqual, 404) // create a manifest - m := ispec.Manifest{Layers: []ispec.Descriptor{{Digest: digest}}} + m := ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: digest, + Size: int64(len(content)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(len(content)), + }, + }, + } + m.SchemaVersion = 2 content, err = json.Marshal(m) So(err, ShouldBeNil) digest = godigest.FromBytes(content) @@ -498,7 +511,20 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) // create a manifest with same blob but a different tag - m = ispec.Manifest{Layers: []ispec.Descriptor{{Digest: digest}}} + m = ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: digest, + Size: int64(len(content)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(len(content)), + }, + }, + } + m.SchemaVersion = 2 content, err = json.Marshal(m) So(err, ShouldBeNil) digest = godigest.FromBytes(content) @@ -599,7 +625,20 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) // create a manifest - m := ispec.Manifest{Layers: []ispec.Descriptor{{Digest: digest}}} + m := ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: digest, + Size: int64(len(content)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(len(content)), + }, + }, + } + m.SchemaVersion = 2 content, err = json.Marshal(m) So(err, ShouldBeNil) digest = godigest.FromBytes(content) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index eaf9637a..44547317 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -15,9 +15,9 @@ import ( "github.com/anuvu/zot/errors" zlog "github.com/anuvu/zot/pkg/log" guuid "github.com/gofrs/uuid" + "github.com/openSUSE/umoci" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/openSUSE/umoci" "github.com/rs/zerolog" ) @@ -335,6 +335,11 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType return "", errors.ErrBadManifest } + if m.SchemaVersion != 2 { + is.log.Error().Int("SchemaVersion", m.SchemaVersion).Msg("invalid manifest") + return "", errors.ErrBadManifest + } + for _, l := range m.Layers { digest := l.Digest blobPath := is.BlobPath(repo, digest) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 0419104f..f194af89 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -105,6 +105,7 @@ func TestAPIs(t *testing.T) { So(err, ShouldBeNil) m := ispec.Manifest{} + m.SchemaVersion = 2 mb, _ := json.Marshal(m) Convey("Bad image manifest", func() { @@ -116,7 +117,20 @@ func TestAPIs(t *testing.T) { }) Convey("Good image manifest", func() { - m := ispec.Manifest{Layers: []ispec.Descriptor{{Digest: d}}} + m := ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 mb, _ = json.Marshal(m) d := godigest.FromBytes(mb) _, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb) From 2fd87b6a8617f7f9e7d03b23cc2f5a6ca6ad5fb5 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 20 Mar 2020 10:56:19 -0700 Subject: [PATCH 6/7] pkg/api: use a rwlock when accessing storage The original patch used a mutex, however, the workload patterns are likely to be read-heavy, so use a rwlock instead. --- pkg/api/routes.go | 26 ++++++++++++++++++-------- pkg/storage/storage.go | 4 ++-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 02e2722c..24bc5588 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -42,17 +42,27 @@ const ( type RouteHandler struct { c *Controller - blobLock sync.Mutex + blobLock sync.RWMutex } func NewRouteHandler(c *Controller) *RouteHandler { - rh := &RouteHandler{c: c, blobLock: sync.Mutex{}} + rh := &RouteHandler{c: c, blobLock: sync.RWMutex{}} rh.SetupRoutes() return rh } -func (rh *RouteHandler) blobLockWrapper(f func (w http.ResponseWriter, r *http.Request)) func (w http.ResponseWriter, r *http.Request) { +// blobRLockWrapper calls the real handler with read-lock held +func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + rh.blobLock.RLock() + f(w, r) + rh.blobLock.RUnlock() + } +} + +// blobLockWrapper calls the real handler with write-lock held +func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { rh.blobLock.Lock() f(w, r) @@ -67,23 +77,23 @@ func (rh *RouteHandler) SetupRoutes() { g.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", NameRegexp.String()), rh.ListTags).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.CheckManifest).Methods("HEAD") + rh.blobRLockWrapper(rh.CheckManifest)).Methods("HEAD") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.GetManifest).Methods("GET") + rh.blobRLockWrapper(rh.GetManifest)).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), rh.blobLockWrapper(rh.UpdateManifest)).Methods("PUT") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), rh.blobLockWrapper(rh.DeleteManifest)).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), - rh.CheckBlob).Methods("HEAD") + rh.blobRLockWrapper(rh.CheckBlob)).Methods("HEAD") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), - rh.GetBlob).Methods("GET") + rh.blobRLockWrapper(rh.GetBlob)).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), rh.blobLockWrapper(rh.DeleteBlob)).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()), rh.blobLockWrapper(rh.CreateBlobUpload)).Methods("POST") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.GetBlobUpload).Methods("GET") + rh.blobRLockWrapper(rh.GetBlobUpload)).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), rh.blobLockWrapper(rh.PatchBlobUpload)).Methods("PATCH") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 44547317..7da82df1 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -35,7 +35,7 @@ type BlobUpload struct { // ImageStore provides the image storage operations. type ImageStore struct { rootDir string - lock *sync.Mutex + lock *sync.RWMutex blobUploads map[string]BlobUpload log zerolog.Logger } @@ -43,7 +43,7 @@ type ImageStore struct { // NewImageStore returns a new image store backed by a file storage. func NewImageStore(rootDir string, log zlog.Logger) *ImageStore { is := &ImageStore{rootDir: rootDir, - lock: &sync.Mutex{}, + lock: &sync.RWMutex{}, blobUploads: make(map[string]BlobUpload), log: log.With().Caller().Logger(), } From dfe023225f8931ac5804c97e82334d1be26f7f27 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 20 Mar 2020 10:56:19 -0700 Subject: [PATCH 7/7] pkg/api: use a rwlock when accessing storage The original patch used a mutex, however, the workload patterns are likely to be read-heavy, so use a rwlock instead. --- WORKSPACE | 4 ++-- pkg/api/routes.go | 6 ++++-- pkg/storage/storage_test.go | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 0c6232d9..c26fbd20 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1362,8 +1362,8 @@ go_repository( go_repository( name = "com_github_opensuse_umoci", importpath = "github.com/openSUSE/umoci", - sum = "h1:MZgFLy5Jl3VKe5uEOU9c25FoySbx3vUXeXLw4Jf6aRs=", - version = "v0.4.5", + sum = "h1:/8Yu54FufyHHQgIZ/wLy+BLQyzk0gbOG24xf5suWOOI=", + version = "v0.4.6-0.20200320140503-9aa268eeb258", ) go_repository( diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 24bc5588..75e0b2ee 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -53,7 +53,8 @@ func NewRouteHandler(c *Controller) *RouteHandler { } // blobRLockWrapper calls the real handler with read-lock held -func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { +func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, + r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { rh.blobLock.RLock() f(w, r) @@ -62,7 +63,8 @@ func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, r *http.R } // blobLockWrapper calls the real handler with write-lock held -func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { +func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter, + r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { rh.blobLock.Lock() f(w, r) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index f194af89..6d017fc5 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -148,7 +148,6 @@ func TestAPIs(t *testing.T) { _, _, _, err = il.GetImageManifest("test", d.String()) So(err, ShouldNotBeNil) }) - }) err = il.DeleteBlobUpload("test", v)