JointCloud
/
JCS-pub
24
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

部署脚本

This commit is contained in:
songjc 2024-05-22 17:05:53 +08:00
parent 1266fbd069
commit b145d9c95f
66 changed files with 3052 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
deploy/build_all.sh Normal file
View File

@ -0,0 +1,46 @@
#!/bin/bash
build_folder="$PWD/build"
imagebuild_folder="$PWD/imagebuild"
yml_folder="$PWD/yml"
echo "开始构建agent镜像..."
cd "$imagebuild_folder"/agent || exit
rm -rf agent
rm -rf confs
cp -r "$build_folder"/agent .
#cp -r "$build_folder"/confs .
sh build.sh
echo "agent镜像构建完成"
echo "开始构建coordinator镜像..."
cd "$imagebuild_folder"/coordinator || exit
rm -rf coordinator
rm -rf confs
cp -r "$build_folder"/coordinator .
#cp -r "$build_folder"/confs .
sh build.sh
echo "coordinator镜像构建完成"
echo "开始构建scanner镜像..."
cd "$imagebuild_folder"/scanner || exit
rm -rf scanner
rm -rf confs
cp -r "$build_folder"/scanner .
#cp -r "$build_folder"/confs .
sh build.sh
echo "scanner镜像构建完成"
echo "开始构建client镜像..."
cd "$imagebuild_folder"/client || exit
rm -rf client
rm -rf confs
cp -r "$build_folder"/client .
#cp -r "$build_folder"/confs .
sh build.sh
echo "client镜像构建完成"
echo "全部镜像构建完成"
#echo "生成yaml脚本"
#cd "$yml_folder" || exit
#sh replace.sh

6
deploy/imagebuild/agent/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
#FROM scratch
FROM alpine:latest
COPY . /opt
WORKDIR /opt/agent
RUN chmod +x agent
ENTRYPOINT ["./agent"]

4
deploy/imagebuild/agent/build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
docker build -t 112.95.163.90:5010/agentservice-x86:latest .
docker push 112.95.163.90:5010/agentservice-x86:latest

6
deploy/imagebuild/client/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
#FROM scratch
FROM alpine:latest
COPY . /opt
WORKDIR /opt/client
RUN chmod +x client
ENTRYPOINT ["./client","serve","http"]

4
deploy/imagebuild/client/build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
docker build -t 112.95.163.90:5010/clientservice-x86:latest .
docker push 112.95.163.90:5010/clientservice-x86:latest

12
deploy/imagebuild/coordinator/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
#FROM scratch
FROM alpine:latest
COPY . /opt
WORKDIR /opt/coordinator
RUN apk add --no-cache tzdata
ENV TZ=Asia/Shanghai
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN chmod +x coordinator
#ENTRYPOINT ["tail","-f","/etc/hosts"]
#RUN ./coordinator &
#ENTRYPOINT ["tail","-f","log/coordinator.log"]
ENTRYPOINT ["./coordinator"]

4
deploy/imagebuild/coordinator/build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
docker build -t 112.95.163.90:5010/coordinatorservice-x86:latest .
docker push 112.95.163.90:5010/coordinatorservice-x86:latest

9
deploy/imagebuild/scanner/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
#FROM scratch
FROM alpine:latest
COPY . /opt
WORKDIR /opt/scanner
RUN apk add --no-cache tzdata
ENV TZ=Asia/Shanghai
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN chmod +x scanner
ENTRYPOINT ["./scanner"]

4
deploy/imagebuild/scanner/build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
docker build -t 112.95.163.90:5010/scannerservice-x86:latest .
docker push 112.95.163.90:5010/scannerservice-x86:latest

14
deploy/readme.md Normal file
View File

@ -0,0 +1,14 @@
1、将storage编译好的build文件放在cloudream目录下
2、在cloudream目录下，运行build_all.sh，构建镜像
./build_all.sh
3、进入yaml目录
cd yaml
4、修改config.ini，填好镜像仓库及其他agent部署信息
5、运行replace.sh，生成yaml文件
6、修改rclone_pv_*.yaml，填写对应节点的rclone配置文件
7、执行启动脚本start.sh，运行pod
./start.sh
8、等待启动完成后，查看pod是否正常运行
kubectl get po -A -owide
9、若需要停止，执行stop.sh
./stop.sh

25
deploy/yaml/config.ini Normal file
View File

@ -0,0 +1,25 @@
[General]
image_registry=112.95.163.90:5010
[agent1]
label=pcm3
port=32001
node=izbp1h7pf0vgxo33ss3b0bz
[agent2]
label=pcm03
port=32002
node=ecs-test
[coordinator]
label=pcm00
node=10-9-72-244
[scanner]
label=pcm01
node=10-9-72-244
[client]
label=pcm2
port=32010
node=10-9-72-244

42
deploy/yaml/config/agent-{{NODE_NAME}}.config.json Normal file
View File

@ -0,0 +1,42 @@
{
"id": 1,
"local": {
"nodeID": 1,
"localIP": "127.0.0.1",
"externalIP": "127.0.0.1",
"locationID": 1
},
"grpc": {
"ip": "127.0.0.1",
"port": 5010
},
"logger": {
"output": "file",
"outputFileName": "agent",
"outputDirectory": "log",
"level": "debug"
},
"rabbitMQ": {
"address": "127.0.0.1:5672",
"account": "cloudream",
"password": "123456",
"vhost": "/"
},
"ipfs": {
"address": "127.0.0.1:5001"
},
"distlock": {
"etcdAddress": "127.0.0.1:2379",
"etcdUsername": "",
"etcdPassword": "",
"etcdLockLeaseTimeSec": 5,
"randomReleasingDelayMs": 3000,
"serviceDescription": "I am a agent"
},
"connectivity": {
"testInterval": 300
},
"downloader": {
"maxStripCacheCount": 100
}
}

35
deploy/yaml/config/client.config.json Normal file
View File

@ -0,0 +1,35 @@
{
"local": {
"localIP": "127.0.0.1",
"externalIP": "127.0.0.1",
"locationID": 1
},
"agentGRPC": {
"port": 5010
},
"logger": {
"output": "stdout",
"level": "debug"
},
"rabbitMQ": {
"address": "127.0.0.1:5672",
"account": "cloudream",
"password": "123456",
"vhost": "/"
},
"ipfs": null,
"distlock": {
"etcdAddress": "127.0.0.1:2379",
"etcdUsername": "",
"etcdPassword": "",
"etcdLockLeaseTimeSec": 5,
"randomReleasingDelayMs": 3000,
"serviceDescription": "I am a client"
},
"connectivity": {
"testInterval": 300
},
"downloader": {
"maxStripCacheCount": 100
}
}

20
deploy/yaml/config/coordinator.config.json Normal file
View File

@ -0,0 +1,20 @@
{
"logger": {
"output": "file",
"outputFileName": "coordinator",
"outputDirectory": "log",
"level": "debug"
},
"db": {
"address": "127.0.0.1:3306",
"account": "root",
"password": "123456",
"databaseName": "cloudream"
},
"rabbitMQ": {
"address": "127.0.0.1:5672",
"account": "cloudream",
"password": "123456",
"vhost": "/"
}
}

26
deploy/yaml/config/rclone_pv_{{NODE_NAME}}.yaml Normal file
View File

@ -0,0 +1,26 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: rclone-{{NODE_NAME}}
labels:
name: rclone-{{NODE_NAME}}
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
storageClassName: rclone
csi:
driver: csi-rclone
volumeHandle: rclone-data-id
volumeAttributes:
remote: "xxxx"
remotePath: "xxxx"
configData: |
[xxxx]
type = s3
provider = xxxx
access_key_id = xxxx
secret_access_key = xxxx
endpoint = xxxx

30
deploy/yaml/config/scanner.config.json Normal file
View File

@ -0,0 +1,30 @@
{
"ecFileSizeThreshold": 104857600,
"nodeUnavailableSeconds": 300,
"logger": {
"output": "file",
"outputFileName": "scanner",
"outputDirectory": "log",
"level": "debug"
},
"db": {
"address": "127.0.0.1:3306",
"account": "root",
"password": "123456",
"databaseName": "cloudream"
},
"rabbitMQ": {
"address": "127.0.0.1:5672",
"account": "cloudream",
"password": "123456",
"vhost": "/"
},
"distlock": {
"etcdAddress": "127.0.0.1:2379",
"etcdUsername": "",
"etcdPassword": "",
"etcdLockLeaseTimeSec": 5,
"randomReleasingDelayMs": 3000,
"serviceDescription": "I am a scanner"
}
}

18
deploy/yaml/csi/build-arm64/Dockerfile Normal file
View File

@ -0,0 +1,18 @@
FROM alpine:3.16
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
RUN apk add --no-cache ca-certificates bash fuse3 curl unzip tini
#RUN curl https://rclone.org/install.sh | bash
COPY rclone /usr/bin/rclone
RUN chmod +x /usr/bin/rclone
# Use pre-compiled version (with cirectory marker patch)
# https://github.com/rclone/rclone/pull/5323
# COPY bin/rclone /usr/bin/rclone
# RUN chmod 755 /usr/bin/rclone \
# && chown root:root /usr/bin/rclone
COPY ./_output/csi-rclone-plugin /bin/csi-rclone-plugin
RUN chmod +x /bin/csi-rclone-plugin
ENTRYPOINT [ "/sbin/tini", "--"]
CMD ["/bin/csi-rclone-plugin"]

4
deploy/yaml/csi/build-arm64/build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
docker build -t 10.16.43.85:5010/csi-rclone-reloaded_arm64:v1.4.0 .
docker push 10.16.43.85:5010/csi-rclone-reloaded_arm64:v1.4.0

18
deploy/yaml/csi/build-x86/Dockerfile Normal file
View File

@ -0,0 +1,18 @@
FROM alpine:3.16
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
RUN apk add --no-cache ca-certificates bash fuse3 curl unzip tini
#RUN curl https://rclone.org/install.sh | bash
COPY rclone /usr/bin/rclone
RUN chmod +x /usr/bin/rclone
# Use pre-compiled version (with cirectory marker patch)
# https://github.com/rclone/rclone/pull/5323
# COPY bin/rclone /usr/bin/rclone
# RUN chmod 755 /usr/bin/rclone \
# && chown root:root /usr/bin/rclone
COPY ./_output/csi-rclone-plugin /bin/csi-rclone-plugin
RUN chmod +x /bin/csi-rclone-plugin
ENTRYPOINT [ "/sbin/tini", "--"]
CMD ["/bin/csi-rclone-plugin"]

4
deploy/yaml/csi/build-x86/build.sh Normal file
View File

@ -0,0 +1,4 @@
#!/bin/bash
docker build -t 112.95.163.90:5010/csi-rclone-reloaded:v1.4.0 .
docker push 112.95.163.90:5010/csi-rclone-reloaded:v1.4.0

48
deploy/yaml/csi/csi-controller-rbac.yaml Normal file
View File

@ -0,0 +1,48 @@
# This YAML file contains RBAC API objects that are necessary to run external
# CSI attacher for rclone adapter
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-controller-rclone
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-controller-rclone
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-attacher-role-rclone
subjects:
- kind: ServiceAccount
name: csi-controller-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: external-controller-rclone
apiGroup: rbac.authorization.k8s.io

56
deploy/yaml/csi/csi-controller-rclone.yaml Normal file
View File

@ -0,0 +1,56 @@
# This YAML file contains attacher & csi driver API objects that are necessary
# to run external CSI attacher for rclone
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: csi-controller-rclone
namespace: kube-system
spec:
serviceName: "csi-controller-rclone"
replicas: 1
selector:
matchLabels:
app: csi-controller-rclone
template:
metadata:
labels:
app: csi-controller-rclone
spec:
serviceAccountName: csi-controller-rclone
containers:
- name: csi-attacher
#image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
image: registry.cn-hangzhou.aliyuncs.com/google_containers/csi-attacher:v3.4.0
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
env:
- name: ADDRESS
value: /csi/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: rclone
image: dvcrn/csi-rclone-reloaded:v1.4.0
image: 112.95.163.90:5010/csi-rclone-reloaded:v1.4.0
args:
- "/bin/csi-rclone-plugin"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /plugin
volumes:
- name: socket-dir
emptyDir: {}

8
deploy/yaml/csi/csi-driver.yaml Normal file
View File

@ -0,0 +1,8 @@
# this should be deregistered once the controller stops
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi-rclone
spec:
attachRequired: true
podInfoOnMount: false # are we sure about this?

40
deploy/yaml/csi/csi-nodeplugin-rbac.yaml Normal file
View File

@ -0,0 +1,40 @@
# This YAML defines all API objects to create RBAC roles for CSI node plugin
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nodeplugin-rclone
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin-rclone
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["secrets", "secret"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin-rclone
subjects:
- kind: ServiceAccount
name: csi-nodeplugin-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-nodeplugin-rclone
apiGroup: rbac.authorization.k8s.io

106
deploy/yaml/csi/csi-nodeplugin-rclone.yaml Normal file
View File

@ -0,0 +1,106 @@
# This YAML file contains driver-registrar & csi driver nodeplugin API objects
# that are necessary to run CSI nodeplugin for rclone
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-nodeplugin-rclone
namespace: kube-system
spec:
selector:
matchLabels:
app: csi-nodeplugin-rclone
template:
metadata:
labels:
app: csi-nodeplugin-rclone
spec:
serviceAccountName: csi-nodeplugin-rclone
#hostNetwork: true
#dnsPolicy: ClusterFirstWithHostNet
dnsPolicy: Default
containers:
- name: node-driver-registrar
#image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
image: registry.cn-hangzhou.aliyuncs.com/google_containers/csi-node-driver-registrar:v2.4.0
lifecycle:
preStop:
exec:
command:
[
"/bin/sh",
"-c",
"rm -rf /registration/csi-rclone /registration/csi-rclone-reg.sock",
]
args:
- --v=5
- --csi-address=/plugin/csi.sock
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-rclone/csi.sock
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /plugin
- name: registration-dir
mountPath: /registration
- name: rclone
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
#image: dvcrn/csi-rclone-reloaded:v1.4.0
image: 112.95.163.90:5010/csi-rclone-reloaded:v1.4.0
args:
- "/bin/csi-rclone-plugin"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
# - "2>&1 > /opt/log/a.log"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
#- name: RCLONE_LOG_FILE
# value: /opt/log/a.log
#- name: RCLONE_LOG_LEVEL
# value: DEBUG
imagePullPolicy: "Always"
lifecycle:
postStart:
exec:
command:
[
"/bin/sh",
"-c",
"mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') ; done",
]
volumeMounts:
- name: plugin-dir
mountPath: /plugin
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
#- name: test-dir
# mountPath: /opt/log
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rclone
type: DirectoryOrCreate
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- hostPath:
path: /var/lib/kubelet/plugins_registry
type: DirectoryOrCreate
name: registration-dir
#- name: test-dir
# hostPath:
# path: /home/pcm/abc
# type: DirectoryOrCreate

66
deploy/yaml/csi/csi-rclone-reloaded-master/.github/workflows/build_images.yml vendored Normal file
View File

@ -0,0 +1,66 @@
---
# Multiarch build file credits go to Lars Kellogg-Stedman at blog.oddbit.com. If You ever see this - thanks!
name: "build images"
on:
push:
branches:
- master
jobs:
docker:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=dvcrn/${GITHUB_REPOSITORY#*/}
VERSION=$(cat VERSION)
TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:latest"
echo ${TAGS}
echo ${VERSION}
echo ${DOCKER_IMAGE}
echo "tags=${TAGS}" >> $GITHUB_ENV
echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v2
with:
username: dvcrn
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build
uses: docker/build-push-action@v2
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ env.tags }}
# - name: Build DM
# uses: docker/build-push-action@v2
# with:
# builder: ${{ steps.buildx.outputs.name }}
# context: .
# file: ./Dockerfile.dm
# platforms: linux/amd64,linux/arm64
# push: true
# tags: ${{ env.tags }}

3
deploy/yaml/csi/csi-rclone-reloaded-master/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
_output/
.vscode
rclone-build/

10
deploy/yaml/csi/csi-rclone-reloaded-master/CHANGELOG.txt Normal file
View File

@ -0,0 +1,10 @@
# CHANGELOG
1.4.0 (dvcrn):
- Merge support for specifying config in secrets: https://github.com/wunderio/csi-rclone/pull/7
- Remove namespace of storageclass
- Move all resources into kube-system namespace
1.3.0:
- Container init changed to tini
- rclone plugin version v1.59.2

23
deploy/yaml/csi/csi-rclone-reloaded-master/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
####
FROM golang:alpine AS builder
RUN apk update && apk add --no-cache git make bash
WORKDIR $GOPATH/src/csi-rclone-nodeplugin
COPY . .
RUN make plugin
####
FROM alpine:3.16
RUN apk add --no-cache ca-certificates bash fuse curl unzip tini
RUN curl https://rclone.org/install.sh | bash
# Use pre-compiled version (with cirectory marker patch)
# https://github.com/rclone/rclone/pull/5323
# COPY bin/rclone /usr/bin/rclone
# RUN chmod 755 /usr/bin/rclone \
# && chown root:root /usr/bin/rclone
COPY --from=builder /go/src/csi-rclone-nodeplugin/_output/csi-rclone-plugin /bin/csi-rclone-plugin
ENTRYPOINT [ "/sbin/tini", "--"]
CMD ["/bin/csi-rclone-plugin"]

23
deploy/yaml/csi/csi-rclone-reloaded-master/Dockerfile.dm Normal file
View File

@ -0,0 +1,23 @@
####
FROM golang:alpine AS builder
RUN apk update && apk add --no-cache git make bash
WORKDIR $GOPATH/src/csi-rclone-nodeplugin
COPY . .
RUN make plugin-dm
####
FROM alpine:3.16
RUN apk add --no-cache ca-certificates bash fuse curl unzip tini
# RUN curl https://rclone.org/install.sh | bash
# Use pre-compiled version (with cirectory marker patch)
# https://github.com/rclone/rclone/pull/5323
COPY ./install-dm.sh /tmp
COPY ./rclone-build /tmp/rclone-build
RUN /tmp/install-dm.sh
COPY --from=builder /go/src/csi-rclone-nodeplugin/_output/csi-rclone-plugin-dm /bin/csi-rclone-plugin
ENTRYPOINT [ "/sbin/tini", "--"]
CMD ["/bin/csi-rclone-plugin"]

202
deploy/yaml/csi/csi-rclone-reloaded-master/LICENSE Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

54
deploy/yaml/csi/csi-rclone-reloaded-master/Makefile Normal file
View File

@ -0,0 +1,54 @@
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
VERSION=$(shell cat VERSION)
REGISTRY_NAME=dvcrn
IMAGE_NAME=csi-rclone-reloaded
IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION)
.PHONY: all rclone-plugin clean rclone-container
all: plugin container push
dm: plugin-dm container-dm push-dm
plugin:
go mod download
CGO_ENABLED=0 GOOS=linux go build -a -gcflags=-trimpath=$(go env GOPATH) -asmflags=-trimpath=$(go env GOPATH) -ldflags '-X github.com/wunderio/csi-rclone/pkg/rclone.DriverVersion=$(VERSION) -extldflags "-static"' -o _output/csi-rclone-plugin ./cmd/csi-rclone-plugin
plugin-dm:
go mod download
CGO_ENABLED=0 GOOS=linux go build -a -gcflags=-trimpath=$(go env GOPATH) -asmflags=-trimpath=$(go env GOPATH) -ldflags '-X github.com/wunderio/csi-rclone/pkg/rclone.DriverVersion=$(VERSION)-dm -extldflags "-static"' -o _output/csi-rclone-plugin-dm ./cmd/csi-rclone-plugin
container:
docker build -t $(IMAGE_TAG) -f ./cmd/csi-rclone-plugin/Dockerfile .
container-dm:
docker build -t $(IMAGE_TAG)-dm -f ./cmd/csi-rclone-plugin/Dockerfile.dm .
push:
docker push $(IMAGE_TAG)
push-dm:
docker push $(IMAGE_TAG)-dm
buildx:
docker buildx build --platform linux/amd64,linux/arm64 --push -t $(IMAGE_TAG) -f ./cmd/csi-rclone-plugin/Dockerfile .
buildx-dm:
docker buildx build --platform linux/amd64,linux/arm64 --push -t $(IMAGE_TAG)-dm -f ./cmd/csi-rclone-plugin/Dockerfile.dm .
clean:
go clean -r -x
-rm -rf _output

185
deploy/yaml/csi/csi-rclone-reloaded-master/README.md Normal file
View File

@ -0,0 +1,185 @@
# CSI rclone mount plugin
Fork of https://github.com/wunderio/csi-rclone that is a bit slow with merging PRs
Differences with that fork:
- Everything is under kube-system namespace
- Allow specifying of secrets for PV
- StorageClass is no longer namespaced
This project implements Container Storage Interface (CSI) plugin that allows using [rclone mount](https://rclone.org/) as storage backend. Rclone mount points and [parameters](https://rclone.org/commands/rclone_mount/) can be configured using Secret or PersistentVolume volumeAttibutes.
## Kubernetes cluster compatability
Works (tested):
- `deploy/kubernetes/1.19`: K8S>= 1.19.x (due to storage.k8s.io/v1 CSIDriver API)
- `deploy/kubernetes/1.13`: K8S 1.13.x - 1.21.x (storage.k8s.io/v1beta1 CSIDriver API)
Does not work:
- v1.12.7-gke.10, driver name csi-rclone not found in the list of registered CSI drivers
## Installing CSI driver to kubernetes cluster
TLDR: ` kubectl apply -f deploy/kubernetes/1.19` (or `deploy/kubernetes/1.13` for older version) to get the CSI setup
### Example: Adding Dropbox through rclone
The easiest way to use this is to specify your rclone configuration inside the PV:
```yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: rclone-dropbox
labels:
name: rclone-dropbox
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 10Gi
storageClassName: rclone
csi:
driver: csi-rclone
volumeHandle: rclone-dropbox-data-id
volumeAttributes:
remote: "dropbox"
remotePath: ""
configData: |
[dropbox]
type = dropbox
client_id = xxx
client_secret = xxx
token = {"access_token":"xxx","token_type":"bearer","refresh_token":"xxx","expiry":"xxx"}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rclone-dropbox
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
storageClassName: rclone
selector:
matchLabels:
name: rclone-dropbox
```
(to get access token, setup Dropbox locally with rclone first, then copy whatever `rclone config show` gives you)
### Example: S3 storage without direct rclone configuration
```yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: rclone-wasabi
labels:
name: rclone-wasabi
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 1000Gi
storageClassName: rclone
csi:
driver: csi-rclone
volumeHandle: data-id
volumeAttributes:
remote: "bucketname"
remotePath: ""
s3-provider: "Wasabi"
s3-endpoint: "https://s3.ap-southeast-1.wasabisys.com"
s3-access-key-id: "xxx"
s3-secret-access-key: "xxx"
---
<pvc manifest here>
```
### Example: Using a secret (thanks to [wunderio/csi-rclone#7](https://github.com/wunderio/csi-rclone/pull/7))
_Note:_ secrets act as defaults, you can still override keys in your PV definitions.
_Note 2_: Use `secret-rclone` as global default for when there are no secrets defined, for example if you always want the same S3 credentials across your PVs
_Note 3_: Secrets need to be in the same namespace as the csi controller, so if you used the default of this repository, add it to `kube-system`
```yaml
apiVersion: v1
kind: Secret
metadata:
name: my-secret
namespace: kube-system # <-- secret needs to be in kube-system namespace, same as CSI controller
type: Opaque
stringData:
remote: "my-s3"
remotePath: "projectname"
configData: |
[my-s3]
type = s3
provider = Minio
access_key_id = ACCESS_KEY_ID
secret_access_key = SECRET_ACCESS_KEY
endpoint = http://minio-release.default:9000
```
Then specify it into the PV:
```yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: rclone-dropbox
labels:
name: rclone-dropbox
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 10Gi
storageClassName: rclone
csi:
driver: csi-rclone
volumeHandle: rclone-dropbox-data-id
volumeAttributes:
secretName: "my-secret"
```
## Debugging & logs
- After creating a pod, if something goes wrong you should be able to see it using `kubectl describe <pod>`
- Check logs of the controller: `kubectl logs -f -l app=csi-nodeplugin-rclone --namespace kube-system -c rclone`
## Building plugin and creating image
Current code is referencing projects repository on github.com. If you fork the repository, you have to change go includes in several places (use search and replace).
1. First push the changed code to remote. The build will use paths from `pkg/` directory.
2. Build the plugin
```
make plugin
```
3. Build the container and inject the plugin into it.
```
make container
```
4. Change docker.io account in `Makefile` and use `make push` to push the image to remote.
```
make push
```
## Changelog
See [CHANGELOG.txt](CHANGELOG.txt)

1
deploy/yaml/csi/csi-rclone-reloaded-master/VERSION Normal file
View File

@ -0,0 +1 @@
v1.4.0

15
deploy/yaml/csi/csi-rclone-reloaded-master/cmd/csi-rclone-plugin/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM alpine:3.16
RUN apk add --no-cache ca-certificates bash fuse curl unzip tini
RUN curl https://rclone.org/install.sh | bash
# Use pre-compiled version (with cirectory marker patch)
# https://github.com/rclone/rclone/pull/5323
# COPY bin/rclone /usr/bin/rclone
# RUN chmod 755 /usr/bin/rclone \
# && chown root:root /usr/bin/rclone
COPY ./_output/csi-rclone-plugin /bin/csi-rclone-plugin
ENTRYPOINT [ "/sbin/tini", "--"]
CMD ["/bin/csi-rclone-plugin"]

15
deploy/yaml/csi/csi-rclone-reloaded-master/cmd/csi-rclone-plugin/Dockerfile.dm Normal file
View File

@ -0,0 +1,15 @@
FROM alpine:3.16
RUN apk add --no-cache ca-certificates bash fuse curl unzip tini
# RUN curl https://rclone.org/install.sh | bash
# Use pre-compiled version (with cirectory marker patch)
# https://github.com/rclone/rclone/pull/5323
COPY ./install-dm.sh /tmp
COPY ./rclone-build /tmp/rclone-build
RUN /tmp/install-dm.sh
COPY ./_output/csi-rclone-plugin-dm /bin/csi-rclone-plugin
ENTRYPOINT [ "/sbin/tini", "--"]
CMD ["/bin/csi-rclone-plugin"]

65
deploy/yaml/csi/csi-rclone-reloaded-master/cmd/csi-rclone-plugin/main.go Normal file
View File

@ -0,0 +1,65 @@
package main
import (
"flag"
"fmt"
"os"
"github.com/spf13/cobra"
"github.com/wunderio/csi-rclone/pkg/rclone"
)
var (
endpoint string
nodeID string
)
func init() {
flag.Set("logtostderr", "true")
}
func main() {
flag.CommandLine.Parse([]string{})
cmd := &cobra.Command{
Use: "rclone",
Short: "CSI based rclone driver",
Run: func(cmd *cobra.Command, args []string) {
handle()
},
}
cmd.Flags().AddGoFlagSet(flag.CommandLine)
cmd.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id")
cmd.MarkPersistentFlagRequired("nodeid")
cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint")
cmd.MarkPersistentFlagRequired("endpoint")
versionCmd := &cobra.Command{
Use: "version",
Short: "Prints information about this version of csi rclone plugin",
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf(`csi-rclone plugin
Version: %s
`, rclone.DriverVersion)
},
}
cmd.AddCommand(versionCmd)
versionCmd.ResetFlags()
cmd.ParseFlags(os.Args[1:])
if err := cmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "%s", err.Error())
os.Exit(1)
}
os.Exit(0)
}
func handle() {
d := rclone.NewDriver(nodeID, endpoint)
d.Run()
}

66
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.13/csi-controller-rbac.yaml Normal file
View File

@ -0,0 +1,66 @@
# This YAML file contains RBAC API objects that are necessary to run external
# CSI attacher for rclone adapter
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-controller-rclone
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-controller-rclone
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-attacher-role-rclone
subjects:
- kind: ServiceAccount
name: csi-controller-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: external-controller-rclone
apiGroup: rbac.authorization.k8s.io
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-cluster-driver-registrar-role
rules:
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csidrivers"]
verbs: ["create", "delete"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create", "list", "watch", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-cluster-driver-registrar-binding
subjects:
- kind: ServiceAccount
name: csi-controller-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-cluster-driver-registrar-role
apiGroup: rbac.authorization.k8s.io

65
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.13/csi-controller-rclone.yaml Normal file
View File

@ -0,0 +1,65 @@
# This YAML file contains attacher & csi driver API objects that are necessary
# to run external CSI attacher for rclone
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: csi-controller-rclone
namespace: kube-system
spec:
serviceName: "csi-controller-rclone"
replicas: 1
selector:
matchLabels:
app: csi-controller-rclone
template:
metadata:
labels:
app: csi-controller-rclone
spec:
serviceAccountName: csi-controller-rclone
containers:
- name: csi-attacher
image: quay.io/k8scsi/csi-attacher:v1.1.1
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /csi/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-cluster-driver-registrar
image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
args:
- "--v=5"
- '--pod-info-mount-version="v1"'
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: rclone
image: dvcrn/csi-rclone-reloaded:v1.4.0
args:
- "/bin/csi-rclone-plugin"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /plugin
volumes:
- name: socket-dir
emptyDir: {}

40
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.13/csi-nodeplugin-rbac.yaml Normal file
View File

@ -0,0 +1,40 @@
# This YAML defines all API objects to create RBAC roles for CSI node plugin
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nodeplugin-rclone
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin-rclone
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["secrets", "secret"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin-rclone
subjects:
- kind: ServiceAccount
name: csi-nodeplugin-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-nodeplugin-rclone
apiGroup: rbac.authorization.k8s.io

92
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml Normal file
View File

@ -0,0 +1,92 @@
# This YAML file contains driver-registrar & csi driver nodeplugin API objects
# that are necessary to run CSI nodeplugin for rclone
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-nodeplugin-rclone
namespace: kube-system
spec:
selector:
matchLabels:
app: csi-nodeplugin-rclone
template:
metadata:
labels:
app: csi-nodeplugin-rclone
spec:
serviceAccountName: csi-nodeplugin-rclone
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: node-driver-registrar
image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
lifecycle:
preStop:
exec:
command:
[
"/bin/sh",
"-c",
"rm -rf /registration/csi-rclone /registration/csi-rclone-reg.sock",
]
args:
- --v=5
- --csi-address=/plugin/csi.sock
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-rclone/csi.sock
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /plugin
- name: registration-dir
mountPath: /registration
- name: rclone
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: dvcrn/csi-rclone-reloaded:v1.4.0
args:
- "/bin/csi-rclone-plugin"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
imagePullPolicy: "Always"
lifecycle:
postStart:
exec:
command:
[
"/bin/sh",
"-c",
"mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') ; done",
]
volumeMounts:
- name: plugin-dir
mountPath: /plugin
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rclone
type: DirectoryOrCreate
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- hostPath:
path: /var/lib/kubelet/plugins_registry
type: DirectoryOrCreate
name: registration-dir

5
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.13/csi-rclone-storageclass.yaml Normal file
View File

@ -0,0 +1,5 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rclone
provisioner: kubernetes.io/no-provisioner

48
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.19/csi-controller-rbac.yaml Normal file
View File

@ -0,0 +1,48 @@
# This YAML file contains RBAC API objects that are necessary to run external
# CSI attacher for rclone adapter
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-controller-rclone
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-controller-rclone
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-attacher-role-rclone
subjects:
- kind: ServiceAccount
name: csi-controller-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: external-controller-rclone
apiGroup: rbac.authorization.k8s.io

54
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.19/csi-controller-rclone.yaml Normal file
View File

@ -0,0 +1,54 @@
# This YAML file contains attacher & csi driver API objects that are necessary
# to run external CSI attacher for rclone
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: csi-controller-rclone
namespace: kube-system
spec:
serviceName: "csi-controller-rclone"
replicas: 1
selector:
matchLabels:
app: csi-controller-rclone
template:
metadata:
labels:
app: csi-controller-rclone
spec:
serviceAccountName: csi-controller-rclone
containers:
- name: csi-attacher
image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
env:
- name: ADDRESS
value: /csi/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: rclone
image: dvcrn/csi-rclone-reloaded:v1.4.0
args:
- "/bin/csi-rclone-plugin"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /plugin
volumes:
- name: socket-dir
emptyDir: {}

8
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.19/csi-driver.yaml Normal file
View File

@ -0,0 +1,8 @@
# this should be deregistered once the controller stops
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi-rclone
spec:
attachRequired: true
podInfoOnMount: false # are we sure about this?

40
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.19/csi-nodeplugin-rbac.yaml Normal file
View File

@ -0,0 +1,40 @@
# This YAML defines all API objects to create RBAC roles for CSI node plugin
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nodeplugin-rclone
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin-rclone
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["secrets", "secret"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin-rclone
subjects:
- kind: ServiceAccount
name: csi-nodeplugin-rclone
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-nodeplugin-rclone
apiGroup: rbac.authorization.k8s.io

92
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml Normal file
View File

@ -0,0 +1,92 @@
# This YAML file contains driver-registrar & csi driver nodeplugin API objects
# that are necessary to run CSI nodeplugin for rclone
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-nodeplugin-rclone
namespace: kube-system
spec:
selector:
matchLabels:
app: csi-nodeplugin-rclone
template:
metadata:
labels:
app: csi-nodeplugin-rclone
spec:
serviceAccountName: csi-nodeplugin-rclone
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: node-driver-registrar
image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
lifecycle:
preStop:
exec:
command:
[
"/bin/sh",
"-c",
"rm -rf /registration/csi-rclone /registration/csi-rclone-reg.sock",
]
args:
- --v=5
- --csi-address=/plugin/csi.sock
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-rclone/csi.sock
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /plugin
- name: registration-dir
mountPath: /registration
- name: rclone
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: dvcrn/csi-rclone-reloaded:v1.4.0
args:
- "/bin/csi-rclone-plugin"
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://plugin/csi.sock
imagePullPolicy: "Always"
lifecycle:
postStart:
exec:
command:
[
"/bin/sh",
"-c",
"mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') ; done",
]
volumeMounts:
- name: plugin-dir
mountPath: /plugin
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/csi-rclone
type: DirectoryOrCreate
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- hostPath:
path: /var/lib/kubelet/plugins_registry
type: DirectoryOrCreate
name: registration-dir

5
deploy/yaml/csi/csi-rclone-reloaded-master/deploy/kubernetes/1.19/csi-rclone-storageclass.yaml Normal file
View File

@ -0,0 +1,5 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rclone
provisioner: kubernetes.io/no-provisioner

72
deploy/yaml/csi/csi-rclone-reloaded-master/example/kubernetes/nginx-example.yaml Normal file
View File

@ -0,0 +1,72 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: data-rclone-example
labels:
name: data-rclone-example
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 10Gi
storageClassName: rclone
csi:
driver: csi-rclone
volumeHandle: data-id
volumeAttributes:
remote: "s3"
remotePath: "projectname/pvname"
s3-provider: "Minio"
s3-endpoint: "http://minio.minio:9000"
s3-access-key-id: "ACCESS_KEY_ID"
s3-secret-access-key: "SECRET_ACCESS_KEY"
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: data-rclone-example
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
storageClassName: rclone
selector:
matchLabels:
name: data-rclone-example
---
apiVersion: v1
kind: Pod
metadata:
name: nginx-example
labels:
run: nginx-example
spec:
containers:
- image: nginx
imagePullPolicy: Always
name: nginx-example
ports:
- containerPort: 80
protocol: TCP
volumeMounts:
- mountPath: /usr/share/nginx/html
name: data-rclone-example
volumes:
- name: data-rclone-example
persistentVolumeClaim:
claimName: data-rclone-example
---
apiVersion: v1
kind: Service
metadata:
name: nginx-example
labels:
run: nginx-example
spec:
ports:
- port: 80
protocol: TCP
selector:
run: nginx-example

13
deploy/yaml/csi/csi-rclone-reloaded-master/example/kubernetes/rclone-secret-example.yaml Normal file
View File

@ -0,0 +1,13 @@
apiVersion: v1
kind: Secret
metadata:
name: rclone-secret
namespace: kube-system
type: Opaque
stringData:
remote: "s3"
remotePath: "projectname"
s3-provider: "Minio"
s3-endpoint: "http://minio.munio:9000"
s3-access-key-id: "ACCESS_KEY_ID"
s3-secret-access-key: "SECRET_ACCESS_KEY"

16
deploy/yaml/csi/csi-rclone-reloaded-master/example/kubernetes/rclone-secret-file-config.yaml Normal file
View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: Secret
metadata:
name: rclone-secret
namespace: kube-system
type: Opaque
stringData:
remote: "my-s3"
remotePath: "projectname"
configData: |
[my-s3]
type = s3
provider = Minio
access_key_id = ACCESS_KEY_ID
secret_access_key = SECRET_ACCESS_KEY
endpoint = http://minio-release.default:9000

53
deploy/yaml/csi/csi-rclone-reloaded-master/go.mod Normal file
View File

@ -0,0 +1,53 @@
module github.com/wunderio/csi-rclone
go 1.15
require (
github.com/container-storage-interface/spec v1.0.0
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/gogo/protobuf v1.2.1 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
github.com/googleapis/gnostic v0.2.0 // indirect
github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
github.com/hashicorp/golang-lru v0.5.0 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.5 // indirect
github.com/kubernetes-csi/csi-lib-utils v0.3.1 // indirect
github.com/kubernetes-csi/drivers v1.0.2
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/onsi/ginkgo v1.7.0 // indirect
github.com/onsi/gomega v1.4.3 // indirect
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/pborman/uuid v1.2.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/prometheus/client_golang v0.9.2 // indirect
github.com/spf13/afero v1.2.1 // indirect
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3 // indirect
github.com/stretchr/objx v0.1.1 // indirect
github.com/stretchr/testify v1.3.0 // indirect
golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f // indirect
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
google.golang.org/grpc v1.18.0
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.2.2 // indirect
k8s.io/api v0.0.0-20190111032252-67edc246be36
k8s.io/apiextensions-apiserver v0.0.0-20190111034747-7d26de67f177 // indirect
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93
k8s.io/apiserver v0.0.0-20190111033246-d50e9ac5404f // indirect
k8s.io/client-go v10.0.0+incompatible
k8s.io/cloud-provider v0.0.0-20190223141949-e954a34baf43 // indirect
k8s.io/csi-api v0.0.0-20190223140843-b4e64dae0b19 // indirect
k8s.io/klog v0.2.0
k8s.io/kube-openapi v0.0.0-20190222203931-aa8624f5a2df // indirect
k8s.io/kubernetes v1.13.2
k8s.io/utils v0.0.0-20190221042446-c2654d5206da // indirect
sigs.k8s.io/yaml v1.1.0 // indirect
)

155
deploy/yaml/csi/csi-rclone-reloaded-master/go.sum Normal file
View File

@ -0,0 +1,155 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E=
github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q=
github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kubernetes-csi/csi-lib-utils v0.3.1 h1:EPE7WgaMx8XwfBIdxJns3B87V0x8TN1mWoZOVNliUaM=
github.com/kubernetes-csi/csi-lib-utils v0.3.1/go.mod h1:GVmlUmxZ+SUjVLXicRFjqWUUvWez0g0Y78zNV9t7KfQ=
github.com/kubernetes-csi/drivers v1.0.2 h1:kaEAMfo+W5YFr23yedBIY+NGnNjr6/PbPzx7N4GYgiQ=
github.com/kubernetes-csi/drivers v1.0.2/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f h1:qWFY9ZxP3tfI37wYIs/MnIAqK0vlXp1xnYEa5HxFSSY=
golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA=
google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.0.0-20190111032252-67edc246be36 h1:XrFGq/4TDgOxYOxtNROTyp2ASjHjBIITdk/+aJD+zyY=
k8s.io/api v0.0.0-20190111032252-67edc246be36/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
k8s.io/apiextensions-apiserver v0.0.0-20190111034747-7d26de67f177 h1:jtIDnyMLAy15hJmcjRMq3ia0LwHkQBLVo1IRXdDMS38=
k8s.io/apiextensions-apiserver v0.0.0-20190111034747-7d26de67f177/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb23EwaLY85hoAV4SpXfdao=
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
k8s.io/apiserver v0.0.0-20190111033246-d50e9ac5404f h1:jOhsBtH52EgxnCNJrCuToXFfQtb3nQDoBPzItfPmSsI=
k8s.io/apiserver v0.0.0-20190111033246-d50e9ac5404f/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w=
k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34=
k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
k8s.io/cloud-provider v0.0.0-20190223141949-e954a34baf43 h1:JRJFItrctyI8kGWwa9sqZ34eX84t3/D5fO119I4x3Cw=
k8s.io/cloud-provider v0.0.0-20190223141949-e954a34baf43/go.mod h1:LlIffnLBu+GG7d4ppPzC8UnA1Ex8S+ntmSRVsnr7Xy4=
k8s.io/csi-api v0.0.0-20190223140843-b4e64dae0b19 h1:RmLaI0waIeLhtsOQl8ekcCgbHRscA3+nQGSFf3qkbeA=
k8s.io/csi-api v0.0.0-20190223140843-b4e64dae0b19/go.mod h1:GH854hXKH+vaEO06X/DMiE/o3rVO1aw8dXJJpP7awjA=
k8s.io/klog v0.2.0 h1:0ElL0OHzF3N+OhoJTL0uca20SxtYt4X4+bzHeqrB83c=
k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/kube-openapi v0.0.0-20190222203931-aa8624f5a2df h1:htvtrqyyVqMSYjR5bmmLx7RVu2+Rp5o1RVOc51x2YrQ=
k8s.io/kube-openapi v0.0.0-20190222203931-aa8624f5a2df/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
k8s.io/kubernetes v1.13.2 h1:rBz6dubDY4bfv85G6zo04v9G5wniTxvBI9yQ/QxJS3g=
k8s.io/kubernetes v1.13.2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4=
k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

170
deploy/yaml/csi/csi-rclone-reloaded-master/install-dm.sh Normal file
View File

@ -0,0 +1,170 @@
#!/usr/bin/env bash
# error codes
# 0 - exited without problems
# 1 - parameters not supported were used or some unexpected error occurred
# 2 - OS not supported by this script
# 3 - installed version of rclone is up to date
# 4 - supported unzip tools are not available
set -e
#when adding a tool to the list make sure to also add its corresponding command further in the script
unzip_tools_list=('unzip' '7z' 'busybox')
usage() { echo "Usage: sudo -v ; sudo bash install-dm.sh" 1>&2; exit 1; }
#check for beta flag
if [ -n "$1" ] && [ "$1" != "beta" ]; then
usage
fi
#create tmp directory and move to it with macOS compatibility fallback
tmp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'rclone-install.XXXXXXXXXX')
cd "$tmp_dir"
#make sure unzip tool is available and choose one to work with
set +e
for tool in ${unzip_tools_list[*]}; do
trash=$(hash "$tool" 2>>errors)
if [ "$?" -eq 0 ]; then
unzip_tool="$tool"
break
fi
done
set -e
# exit if no unzip tools available
if [ -z "$unzip_tool" ]; then
printf "\nNone of the supported tools for extracting zip archives (${unzip_tools_list[*]}) were found. "
printf "Please install one of them and try again.\n\n"
exit 4
fi
# Make sure we don't create a root owned .config/rclone directory #2127
export XDG_CONFIG_HOME=config
#detect the platform
OS="$(uname)"
case $OS in
Linux)
OS='linux'
;;
FreeBSD)
OS='freebsd'
;;
NetBSD)
OS='netbsd'
;;
OpenBSD)
OS='openbsd'
;;
Darwin)
OS='osx'
binTgtDir=/usr/local/bin
man1TgtDir=/usr/local/share/man/man1
;;
SunOS)
OS='solaris'
echo 'OS not supported'
exit 2
;;
*)
echo 'OS not supported'
exit 2
;;
esac
OS_type="$(uname -m)"
case "$OS_type" in
x86_64|amd64)
OS_type='amd64'
;;
i?86|x86)
OS_type='386'
;;
aarch64|arm64)
OS_type='arm64'
;;
arm*)
OS_type='arm'
;;
*)
echo 'OS type not supported'
exit 2
;;
esac
#download and unzip
rclone_zip="/tmp/rclone-build/rclone-current-${OS}-${OS_type}.zip"
unzip_dir="tmp_unzip_dir_for_rclone"
# there should be an entry in this switch for each element of unzip_tools_list
case "$unzip_tool" in
'unzip')
unzip -a "$rclone_zip" -d "$unzip_dir"
;;
'7z')
7z x "$rclone_zip" "-o$unzip_dir"
;;
'busybox')
mkdir -p "$unzip_dir"
busybox unzip "$rclone_zip" -d "$unzip_dir"
;;
esac
cd $unzip_dir/*
#mounting rclone to environment
case "$OS" in
'linux')
#binary
cp rclone /usr/bin/rclone.new
chmod 755 /usr/bin/rclone.new
chown root:root /usr/bin/rclone.new
mv /usr/bin/rclone.new /usr/bin/rclone
#manual
if ! [ -x "$(command -v mandb)" ]; then
echo 'mandb not found. The rclone man docs will not be installed.'
else
mkdir -p /usr/local/share/man/man1
cp rclone.1 /usr/local/share/man/man1/
mandb
fi
;;
'freebsd'|'openbsd'|'netbsd')
#binary
cp rclone /usr/bin/rclone.new
chown root:wheel /usr/bin/rclone.new
mv /usr/bin/rclone.new /usr/bin/rclone
#manual
mkdir -p /usr/local/man/man1
cp rclone.1 /usr/local/man/man1/
makewhatis
;;
'osx')
#binary
mkdir -m 0555 -p ${binTgtDir}
cp rclone ${binTgtDir}/rclone.new
mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
chmod a=x ${binTgtDir}/rclone
#manual
mkdir -m 0555 -p ${man1TgtDir}
cp rclone.1 ${man1TgtDir}
chmod a=r ${man1TgtDir}/rclone.1
;;
*)
echo 'OS not supported'
exit 2
esac
#update version variable post install
version=$(rclone --version 2>>errors | head -n 1)
printf "\n${version} has successfully installed."
printf '\nNow run "rclone config" for setup. Check https://rclone.org/docs/ for more details.\n\n'
exit 0

50
deploy/yaml/csi/csi-rclone-reloaded-master/pkg/rclone/driver.go Normal file
View File

@ -0,0 +1,50 @@
package rclone
import (
"github.com/container-storage-interface/spec/lib/go/csi"
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
"k8s.io/klog"
)
type driver struct {
csiDriver *csicommon.CSIDriver
endpoint string
ns *nodeServer
cap []*csi.VolumeCapability_AccessMode
cscap []*csi.ControllerServiceCapability
}
var (
DriverName = "csi-rclone"
DriverVersion = "latest"
)
func NewDriver(nodeID, endpoint string) *driver {
klog.Infof("Starting new %s driver in version %s", DriverName, DriverVersion)
d := &driver{}
d.endpoint = endpoint
d.csiDriver = csicommon.NewCSIDriver(DriverName, DriverVersion, nodeID)
d.csiDriver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER})
d.csiDriver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN})
return d
}
func NewNodeServer(d *driver) *nodeServer {
return &nodeServer{
DefaultNodeServer: csicommon.NewDefaultNodeServer(d.csiDriver),
}
}
func (d *driver) Run() {
s := csicommon.NewNonBlockingGRPCServer()
s.Start(d.endpoint,
csicommon.NewDefaultIdentityServer(d.csiDriver),
csicommon.NewDefaultControllerServer(d.csiDriver),
NewNodeServer(d))
s.Wait()
}

25
deploy/yaml/csi/csi-rclone-reloaded-master/pkg/rclone/k8sClient.go Normal file
View File

@ -0,0 +1,25 @@
package rclone
import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
var clientset *kubernetes.Clientset
func GetK8sClient() (*kubernetes.Clientset, error) {
if clientset != nil {
return clientset, nil
}
config, e := rest.InClusterConfig()
if e != nil {
return nil, e
}
clientset, e = kubernetes.NewForConfig(config)
if e != nil {
return nil, e
}
return clientset, nil
}

326
deploy/yaml/csi/csi-rclone-reloaded-master/pkg/rclone/nodeserver.go Normal file
View File

@ -0,0 +1,326 @@
package rclone
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"strings"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog"
"github.com/container-storage-interface/spec/lib/go/csi"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume/util"
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
)
type nodeServer struct {
*csicommon.DefaultNodeServer
mounter *mount.SafeFormatAndMount
}
type mountPoint struct {
VolumeId string
MountPath string
}
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
klog.Infof("NodePublishVolume: called with args %+v", *req)
targetPath := req.GetTargetPath()
notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(targetPath, 0750); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
notMnt = true
} else {
return nil, status.Error(codes.Internal, err.Error())
}
}
if !notMnt {
// testing original mount point, make sure the mount link is valid
if _, err := ioutil.ReadDir(targetPath); err == nil {
klog.Infof("already mounted to target %s", targetPath)
return &csi.NodePublishVolumeResponse{}, nil
}
// todo: mount link is invalid, now unmount and remount later (built-in functionality)
klog.Warningf("ReadDir %s failed with %v, unmount this directory", targetPath, err)
ns.mounter = &mount.SafeFormatAndMount{
Interface: mount.New(""),
Exec: mount.NewOsExec(),
}
if err := ns.mounter.Unmount(targetPath); err != nil {
klog.Errorf("Unmount directory %s failed with %v", targetPath, err)
return nil, err
}
}
mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags()
if req.GetReadonly() {
mountOptions = append(mountOptions, "ro")
}
remote, remotePath, configData, flags, e := extractFlags(req.GetVolumeContext())
if e != nil {
klog.Warningf("storage parameter error: %s", e)
return nil, e
}
e = Mount(remote, remotePath, targetPath, configData, flags)
if e != nil {
if os.IsPermission(e) {
return nil, status.Error(codes.PermissionDenied, e.Error())
}
if strings.Contains(e.Error(), "invalid argument") {
return nil, status.Error(codes.InvalidArgument, e.Error())
}
return nil, status.Error(codes.Internal, e.Error())
}
return &csi.NodePublishVolumeResponse{}, nil
}
// extractFlags extracts the flags from the given volumeContext
// Retturns: remote, remotePath, configData, flags, error
func extractFlags(volumeContext map[string]string) (string, string, string, map[string]string, error) {
// Load default connection settings from secret
var secret *v1.Secret
if secretName, ok := volumeContext["secretName"]; ok {
// Load the secret that the PV spec defines
var e error
secret, e = getSecret(secretName)
if e != nil {
// if the user explicitly requested a secret and there is an error fetching it, bail with an error
return "", "", "", nil, e
}
} else {
// use rclone-secret as the default secret if none was defined
secret, _ = getSecret("rclone-secret")
}
// Empty argument list
flags := make(map[string]string)
// Secret values are default, gets merged and overriden by corresponding PV values
if secret != nil && secret.Data != nil && len(secret.Data) > 0 {
// Needs byte to string casting for map values
for k, v := range secret.Data {
flags[k] = string(v)
}
} else {
klog.Infof("No csi-rclone connection defaults secret found.")
}
if len(volumeContext) > 0 {
for k, v := range volumeContext {
flags[k] = v
}
}
if e := validateFlags(flags); e != nil {
return "", "", "", flags, e
}
remote := flags["remote"]
remotePath := flags["remotePath"]
if remotePathSuffix, ok := flags["remotePathSuffix"]; ok {
remotePath = remotePath + remotePathSuffix
delete(flags, "remotePathSuffix")
}
configData := ""
ok := false
if configData, ok = flags["configData"]; ok {
delete(flags, "configData")
}
delete(flags, "remote")
delete(flags, "remotePath")
delete(flags, "secretName")
return remote, remotePath, configData, flags, nil
}
func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
klog.Infof("NodeUnPublishVolume: called with args %+v", *req)
targetPath := req.GetTargetPath()
if len(targetPath) == 0 {
return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided")
}
m := mount.New("")
notMnt, err := m.IsLikelyNotMountPoint(targetPath)
if err != nil && !mount.IsCorruptedMnt(err) {
return nil, status.Error(codes.Internal, err.Error())
}
if notMnt && !mount.IsCorruptedMnt(err) {
klog.Infof("Volume not mounted")
} else {
err = util.UnmountPath(req.GetTargetPath(), m)
if err != nil {
klog.Infof("Error while unmounting path: %s", err)
// This will exit and fail the NodeUnpublishVolume making it to retry unmount on the next api schedule trigger.
// Since we mount the volume with allow-non-empty now, we could skip this one too.
return nil, status.Error(codes.Internal, err.Error())
}
klog.Infof("Volume %s unmounted successfully", req.VolumeId)
}
return &csi.NodeUnpublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
klog.Infof("NodeUnstageVolume: called with args %+v", *req)
return &csi.NodeUnstageVolumeResponse{}, nil
}
func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
klog.Infof("NodeStageVolume: called with args %+v", *req)
return &csi.NodeStageVolumeResponse{}, nil
}
func validateFlags(flags map[string]string) error {
if _, ok := flags["remote"]; !ok {
return status.Errorf(codes.InvalidArgument, "missing volume context value: remote")
}
if _, ok := flags["remotePath"]; !ok {
return status.Errorf(codes.InvalidArgument, "missing volume context value: remotePath")
}
return nil
}
func getSecret(secretName string) (*v1.Secret, error) {
clientset, e := GetK8sClient()
if e != nil {
return nil, status.Errorf(codes.Internal, "can not create kubernetes client: %s", e)
}
kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
clientcmd.NewDefaultClientConfigLoadingRules(),
&clientcmd.ConfigOverrides{},
)
namespace, _, err := kubeconfig.Namespace()
if err != nil {
return nil, status.Errorf(codes.Internal, "can't get current namespace, error %s", secretName, err)
}
klog.Infof("Loading csi-rclone connection defaults from secret %s/%s", namespace, secretName)
secret, e := clientset.CoreV1().
Secrets(namespace).
Get(secretName, metav1.GetOptions{})
if e != nil {
return nil, status.Errorf(codes.Internal, "can't load csi-rclone settings from secret %s: %s", secretName, e)
}
return secret, nil
}
// Mount routine.
func Mount(remote string, remotePath string, targetPath string, configData string, flags map[string]string) error {
mountCmd := "rclone"
mountArgs := []string{}
defaultFlags := map[string]string{}
defaultFlags["cache-info-age"] = "72h"
defaultFlags["cache-chunk-clean-interval"] = "15m"
defaultFlags["dir-cache-time"] = "5s"
defaultFlags["vfs-cache-mode"] = "writes"
defaultFlags["allow-non-empty"] = "true"
defaultFlags["allow-other"] = "true"
remoteWithPath := fmt.Sprintf(":%s:%s", remote, remotePath)
if strings.Contains(configData, "["+remote+"]") {
remoteWithPath = fmt.Sprintf("%s:%s", remote, remotePath)
klog.Infof("remote %s found in configData, remoteWithPath set to %s", remote, remoteWithPath)
}
// rclone mount remote:path /path/to/mountpoint [flags]
mountArgs = append(
mountArgs,
"mount",
remoteWithPath,
targetPath,
"--daemon",
)
// If a custom flag configData is defined,
// create a temporary file, fill it with configData content,
// and run rclone with --config <tmpfile> flag
if configData != "" {
configFile, err := ioutil.TempFile("", "rclone.conf")
if err != nil {
return err
}
// Normally, a defer os.Remove(configFile.Name()) should be placed here.
// However, due to a rclone mount --daemon flag, rclone forks and creates a race condition
// with this nodeplugin proceess. As a result, the config file gets deleted
// before it's reread by a forked process.
if _, err := configFile.Write([]byte(configData)); err != nil {
return err
}
if err := configFile.Close(); err != nil {
return err
}
mountArgs = append(mountArgs, "--config", configFile.Name())
}
// Add default flags
for k, v := range defaultFlags {
// Exclude overriden flags
if _, ok := flags[k]; !ok {
mountArgs = append(mountArgs, fmt.Sprintf("--%s=%s", k, v))
}
}
// Add user supplied flags
for k, v := range flags {
mountArgs = append(mountArgs, fmt.Sprintf("--%s=%s", k, v))
}
// create target, os.Mkdirall is noop if it exists
err := os.MkdirAll(targetPath, 0750)
if err != nil {
return err
}
klog.Infof("executing mount command cmd=%s, remote=%s, targetpath=%s", mountCmd, remoteWithPath, targetPath)
out, err := exec.Command(mountCmd, mountArgs...).CombinedOutput()
if err != nil {
return fmt.Errorf("mounting failed: %v cmd: '%s' remote: '%s' targetpath: %s output: %q",
err, mountCmd, remoteWithPath, targetPath, string(out))
}
return nil
}

5
deploy/yaml/csi/csi-rclone-storageclass.yaml Normal file
View File

@ -0,0 +1,5 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rclone
provisioner: kubernetes.io/no-provisioner

109
deploy/yaml/replace.sh Normal file
View File

@ -0,0 +1,109 @@
#!/bin/bash
# 配置文件路径
config_file="config.ini"
# YAML 模板文件路径
agent_template_file="template/rclone-agent.yaml.template"
rclone_pv_template_file="template/rclone-pv.yaml.template"
coordinator_template_file="template/coordinator.yaml.template"
scanner_template_file="template/scanner.yaml.template"
client_template_file="template/client.yaml.template"
image_registry_address=$(awk -F "=" '/\[General\]/{f=1} f==1 && /image_registry/{print $2; exit}' "$config_file")
# 逐行读取文件
while IFS= read -r line; do
# 判断是否是标题行
if [[ $line == \[*] ]]; then
current_section=$(echo "$line" | tr -d '[]')
case $current_section in
agent*)
echo "读取agent配置"
while IFS= read -r next_line && [[ $next_line ]]; do
case "$next_line" in
label=*) label=$(echo "$next_line" | cut -d '=' -f 2) ;;
port=*) port=$(echo "$next_line" | cut -d '=' -f 2) ;;
node=*) node=$(echo "$next_line" | cut -d '=' -f 2) ;;
esac
done
if [[ -n $label && -n $port && -n $node ]]; then
# 创建并替换 YAML 文件
agent_yaml_file="rclone_agent_$label.yaml"
#pv_yaml_file="rclone_pv_$label.yaml"
if [ ! -f "$agent_yaml_file" ]; then
sed "s/{{NODE_NAME}}/$label/g;
s/{{NODE_PORT}}/$port/g;
s/{{NODE_ID}}/$node/g;
s/{{IMAGE_REGISTRY_ADDRESS}}/$image_registry_address/g" "$agent_template_file" > "$agent_yaml_file"
fi
#if [ ! -f "$pv_yaml_file" ]; then
# sed "s/{{NODE_NAME}}/$label/g" "$rclone_pv_template_file" > "$pv_yaml_file"
#fi
# 清空变量，以便下一个节点的处理
label=""
port=""
node=""
fi
;;
coordinator)
echo "读取coordinator配置"
while IFS= read -r next_line && [[ $next_line ]]; do
case "$next_line" in
label=*) coor_label=$(echo "$next_line" | cut -d '=' -f 2) ;;
node=*) coor_node=$(echo "$next_line" | cut -d '=' -f 2) ;;
esac
done
if [[ -n $coor_label && -n $coor_node ]]; then
coor_yaml_file="coordinator.yaml"
if [ ! -f "$coor_yaml_file" ]; then
sed "s/{{NODE_NAME}}/$coor_label/g;
s/{{IMAGE_REGISTRY_ADDRESS}}/$image_registry_address/g" "$coordinator_template_file" > "$coor_yaml_file"
fi
fi
;;
scanner)
echo "读取scanner配置"
while IFS= read -r next_line && [[ $next_line ]]; do
case "$next_line" in
label=*) scan_label=$(echo "$next_line" | cut -d '=' -f 2) ;;
node=*) scan_node=$(echo "$next_line" | cut -d '=' -f 2) ;;
esac
done
if [[ -n $scan_label && -n $scan_node ]]; then
scan_yaml_file="scanner.yaml"
if [ ! -f "$scan_yaml_file" ]; then
sed "s/{{NODE_NAME}}/$scan_label/g;
s/{{IMAGE_REGISTRY_ADDRESS}}/$image_registry_address/g" "$scanner_template_file" > "$scan_yaml_file"
fi
fi
;;
client)
echo "读取client配置"
while IFS= read -r next_line && [[ $next_line ]]; do
case "$next_line" in
label=*) cli_label=$(echo "$next_line" | cut -d '=' -f 2) ;;
port=*) cli_port=$(echo "$next_line" | cut -d '=' -f 2) ;;
node=*) cli_node=$(echo "$next_line" | cut -d '=' -f 2) ;;
esac
done
if [[ -n $cli_label && -n $cli_port && -n $cli_node ]]; then
cli_yaml_file="client.yaml"
if [ ! -f "$cli_yaml_file" ]; then
sed "s/{{NODE_NAME}}/$cli_label/g;
s/{{NODE_PORT}}/$cli_port/g;
s/{{IMAGE_REGISTRY_ADDRESS}}/$image_registry_address/g" "$client_template_file" > "$cli_yaml_file"
fi
fi
;;
esac
fi
done < "$config_file"

41
deploy/yaml/start.sh Normal file
View File

@ -0,0 +1,41 @@
#!/bin/bash
#!/bin/bash
# 获取当前路径
current_path=$(pwd)
# 拉起configmap
cd $current_path/config
config_files=$(ls *.json 2>/dev/null)
if [ -z "$config_files" ]; then
echo "当前路径下没有.config.json文件。"
exit 1
fi
for file in $config_files; do
if [[ -f "$file" ]]; then
name=$(echo "$file" | cut -d '.' -f1)
service_name=$(echo "$name" | cut -d '-' -f1)
kubectl create cm $name-config --from-file=$service_name.config.json=./$file
fi
done
# 拉起pod
pv_yaml_files=$(ls rclone_pv_*.yaml 2>/dev/null)
for pv_yaml_file in $pv_yaml_files; do
echo "Applying $pv_yaml_file ..."
kubectl apply -f $pv_yaml_file
done
cd $current_path
yaml_files=$(ls *.yaml 2>/dev/null)
for yaml_file in $yaml_files; do
echo "Applying $yaml_file ..."
kubectl apply -f $yaml_file
done

38
deploy/yaml/stop.sh Normal file
View File

@ -0,0 +1,38 @@
#!/bin/bash
# 获取当前路径
current_path=$(pwd)
# 删除configmap
cd $current_path/config
config_files=$(ls *.json 2>/dev/null)
if [ -z "$config_files" ]; then
echo "当前路径下没有.config.json文件。"
exit 1
fi
for file in $config_files; do
if [[ -f "$file" ]]; then
name=$(echo "$file" | cut -d '.' -f1)
kubectl delete cm $name-config
fi
done
# 删除pod
cd $current_path
yaml_files=$(ls *.yaml 2>/dev/null)
for yaml_file in $yaml_files; do
echo "Delete $yaml_file ..."
kubectl delete -f $yaml_file
done
# 删除pv
cd $current_path/config
pv_yaml_files=$(ls *.yaml 2>/dev/null)
for pv_yaml_file in $pv_yaml_files; do
echo "Delete $pv_yaml_file ..."
kubectl delete -f $pv_yaml_file
done

49
deploy/yaml/template/client.yaml.template Normal file
View File

@ -0,0 +1,49 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: client
name: client
namespace: default
spec:
selector:
matchLabels:
app: client
template:
metadata:
labels:
app: client
spec:
containers:
- name: clientservice
image: {{IMAGE_REGISTRY_ADDRESS}}/clientservice-x86:latest
imagePullPolicy: Always
volumeMounts:
- name: clientconfig
mountPath: /opt/confs
volumes:
- name: clientconfig
configMap:
name: client-config
nodeSelector:
nodetype: {{NODE_NAME}}
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
labels:
app: client
name: client
namespace: default
spec:
ports:
- port: 7890
protocol: TCP
targetPort: 7890
nodePort: {{NODE_PORT}}
selector:
app: client
type: NodePort

31
deploy/yaml/template/coordinator.yaml.template Normal file
View File

@ -0,0 +1,31 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: coordinator
name: coordinator
namespace: default
spec:
selector:
matchLabels:
app: coordinator
template:
metadata:
labels:
app: coordinator
spec:
containers:
- name: coordinatorservice
image: {{IMAGE_REGISTRY_ADDRESS}}/coordinatorservice-x86:latest
imagePullPolicy: Always
volumeMounts:
- name: coordinatorconfig
mountPath: /opt/confs
volumes:
- name: coordinatorconfig
configMap:
name: coordinator-config
nodeSelector:
nodetype: {{NODE_NAME}}
restartPolicy: Always

75
deploy/yaml/template/rclone-agent.yaml.template Normal file
View File

@ -0,0 +1,75 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rclone-{{NODE_NAME}}
spec:
accessModes:
- ReadWriteMany
storageClassName: rclone
resources:
requests:
storage: 10Gi
selector:
matchLabels:
name: rclone-{{NODE_NAME}}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: agent-{{NODE_NAME}}
name: agent-{{NODE_NAME}}
namespace: default
spec:
selector:
matchLabels:
app: agent-{{NODE_NAME}}
template:
metadata:
labels:
app: agent-{{NODE_NAME}}
spec:
containers:
- name: agentservice
image: {{IMAGE_REGISTRY_ADDRESS}}/agentservice-x86:latest
imagePullPolicy: Always
#command: ["tail","-f","/etc/hosts"]
ports:
- containerPort: 5010
protocol: TCP
volumeMounts:
- name: agentconfig
mountPath: /opt/confs
- name: rclone-pvc
mountPath: /opt/storage
volumes:
- name: agentconfig
configMap:
name: agent-{{NODE_NAME}}-config
- name: rclone-pvc
persistentVolumeClaim:
claimName: rclone-{{NODE_NAME}}
dnsPolicy: Default
nodeSelector:
nodetype: {{NODE_NAME}}
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
labels:
app: agent-{{NODE_NAME}}
name: agent-{{NODE_NAME}}
namespace: default
spec:
ports:
- port: 5010
protocol: TCP
targetPort: 5010
nodePort: {{NODE_PORT}}
selector:
app: agent-{{NODE_NAME}}
type: NodePort

26
deploy/yaml/template/rclone-pv.yaml.template Normal file
View File

@ -0,0 +1,26 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: rclone-{{NODE_NAME}}
labels:
name: rclone-{{NODE_NAME}}
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
storageClassName: rclone
csi:
driver: csi-rclone
volumeHandle: rclone-data-id
volumeAttributes:
remote: "xxxx"
remotePath: "xxxx"
configData: |
[xxxx]
type = s3
provider = xxxx
access_key_id = xxxx
secret_access_key = xxxx
endpoint = xxxx

32
deploy/yaml/template/scanner.yaml.template Normal file
View File

@ -0,0 +1,32 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: scanner
name: scanner
namespace: default
spec:
selector:
matchLabels:
app: scanner
template:
metadata:
labels:
app: scanner
spec:
containers:
- name: scannerservice
image: {{IMAGE_REGISTRY_ADDRESS}}/scannerservice-x86:latest
#command: ["tail","-f","/etc/hosts"]
imagePullPolicy: Always
volumeMounts:
- name: scannerconfig
mountPath: /opt/confs
volumes:
- name: scannerconfig
configMap:
name: scanner-config
nodeSelector:
nodetype: {{NODE_NAME}}
restartPolicy: Always