Merge pull request #270 from resouer/upgrade-1.9

Upgrade CNI to 0.6.0 and vendor dockershim to 1.9

.travis.yml

@@ -5,13 +5,17 @@ dist: xenial
 services:
   - docker
 
+env:  
+    global:
+        - DOCKER_VERSION=17.03.2~ce-0~ubuntu-xenial
+
 go:
   - 1.9.x
 
 go_import_path: k8s.io/frakti
 
-before_install:
-  - hack/install-docker.sh
+script:  
+    - docker version
 
 install:
   - docker run -v /usr/local/bin:/target jpetazzo/nsenter

cluster/allinone.sh

@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-FRAKTI_VERSION="v1.1.1"
+FRAKTI_VERSION="v1.9"
 CLUSTER_CIDR="10.244.0.0/16"
 MASTER_CIDR="10.244.1.0/24"
 

cmd/frakti/frakti.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 
 	"github.com/golang/glog"
@@ -30,14 +29,15 @@ import (
 	unikernel "k8s.io/frakti/pkg/unikernel/service"
 	"k8s.io/frakti/pkg/util/flags"
 	"k8s.io/frakti/pkg/util/logs"
+	"k8s.io/frakti/pkg/util/network"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 )
 
 const (
-	fraktiVersion = "1.1.1"
+	fraktiVersion = "1.9"
 
 	// use port 22522 for dockershim streaming
-	privilegedStreamingServerPort = 22522
+	privilegedStreamingServerPort = "22522"
 )
 
 var (
@@ -48,8 +48,8 @@ var (
 		"The endpoint for connecting hyperd, e.g. 127.0.0.1:22318")
 	streamingServerPort = pflag.String("streaming-server-port", "22521",
 		"The port for the streaming server to serve on, e.g. 22521")
-	streamingServerAddress = pflag.String("streaming-server-addr", "0.0.0.0",
-		"The IP address for the streaming server to serve on, e.g. 0.0.0.0")
+	streamingServerAddress = pflag.String("streaming-server-addr", "",
+		"The IP address for the streaming server to serve on, should not be 0.0.0.0 or 127.0.0.1")
 	cniNetDir = pflag.String("cni-net-dir", "/etc/cni/net.d",
 		"The directory for putting cni configuration file")
 	cniPluginDir = pflag.String("cni-plugin-dir", "/opt/cni/bin",
@@ -71,52 +71,46 @@ func main() {
 	defer logs.FlushLogs()
 
 	if *version {
-		glog.Infof("frakti version: %s\n", fraktiVersion)
-		os.Exit(0)
+		glog.Fatalf("frakti version: %s\n", fraktiVersion)
 	}
 
 	if *cgroupDriver != "cgroupfs" && *cgroupDriver != "systemd" {
-		glog.Error("cgroup-driver flag should only be set as 'cgroupfs' or 'systemd'")
-		os.Exit(1)
+		glog.Fatalf("cgroup-driver flag should only be set as 'cgroupfs' or 'systemd'")
 	}
 
 	// 1. Initialize hyper runtime and streaming server
-	streamingConfig := getStreamingConfig()
+	streamingConfig := getStreamingConfig(*streamingServerPort)
 	hyperRuntime, streamingServer, err := hyper.NewHyperRuntime(*hyperEndpoint, streamingConfig, *cniNetDir, *cniPluginDir, *rootDir, *defaultCPUNum, *defaultMemoryMB)
 	if err != nil {
-		glog.Errorf("Initialize hyper runtime failed: %v", err)
-		os.Exit(1)
+		glog.Fatalf("Initialize hyper runtime failed: %v", err)
 	}
 
 	// 2. Initialize privileged runtime and start its own streaming server
 	privilegedRuntime, err := docker.NewPrivilegedRuntimeService(
 		*privilegedRuntimeEndpoint,
-		getprivilegedStreamingConfig(),
+		getStreamingConfig(privilegedStreamingServerPort),
 		*cniNetDir,
 		*cniPluginDir,
 		*cgroupDriver,
 		filepath.Join(*rootDir, "privileged"),
 	)
 	if err != nil && *enablePrivilegedRuntime {
-		glog.Errorf("Initialize privileged runtime failed: %v", err)
-		os.Exit(1)
+		glog.Fatalf("Initialize privileged runtime failed: %v", err)
 	}
 
 	// 3. Initialize unikernel runtime if enabled
 	var unikernelRuntime *unikernel.UnikernelRuntime
 	if *enableUnikernelRuntime {
 		unikernelRuntime, err = unikernel.NewUnikernelRuntimeService(*cniNetDir, *cniPluginDir, *rootDir, *defaultCPUNum, *defaultMemoryMB, *enableUnikernelLog)
 		if err != nil {
-			glog.Errorf("Initialize unikernel runtime failed: %v", err)
-			os.Exit(1)
+			glog.Fatalf("Initialize unikernel runtime failed: %v", err)
 		}
 	}
 
 	// 4. Initialize frakti manager with two runtimes above
 	server, err := manager.NewFraktiManager(hyperRuntime, hyperRuntime, streamingServer, privilegedRuntime, privilegedRuntime, unikernelRuntime, unikernelRuntime)
 	if err != nil {
-		glog.Errorf("Initialize frakti server failed: %v", err)
-		os.Exit(1)
+		glog.Fatalf("Initialize frakti server failed: %v", err)
 	}
 
 	fmt.Println(server.Serve(*listen))
@@ -132,16 +126,21 @@ func generateStreamingConfigInternal() *streaming.Config {
 	}
 }
 
-// Gets the streaming server configuration to use with in-process CRI shims.
-func getStreamingConfig() *streaming.Config {
+// getStreamingConfig returns the streaming server configuration to use with in-process CRI shims.
+func getStreamingConfig(port string) *streaming.Config {
 	config := generateStreamingConfigInternal()
-	config.Addr = fmt.Sprintf("%s:%s", *streamingServerAddress, *streamingServerPort)
-	return config
-}
-
-// Gets the streaming server configuration to use with in-process privileged shims.
-func getprivilegedStreamingConfig() *streaming.Config {
-	config := generateStreamingConfigInternal()
-	config.Addr = fmt.Sprintf("%s:%d", *streamingServerAddress, privilegedStreamingServerPort)
+	var (
+		addr string
+		err  error
+	)
+	if len(*streamingServerAddress) == 0 {
+		addr, err = network.GetLocalIPAddress()
+		if err != nil {
+			glog.Fatalf("failed to get local IP address of host machine: %v", err)
+		}
+	} else {
+		addr = *streamingServerAddress
+	}
+	config.Addr = fmt.Sprintf("%s:%s", addr, port)
 	return config
 }

docs/deploy.md

@@ -93,7 +93,7 @@ systemctl start docker
 #### Install frakti
 
 ```sh
-curl -sSL https://github.com/kubernetes/frakti/releases/download/v1.1.1/frakti -o /usr/bin/frakti
+curl -sSL https://github.com/kubernetes/frakti/releases/download/v1.9/frakti -o /usr/bin/frakti
 chmod +x /usr/bin/frakti
 cgroup_driver=$(docker info | awk '/Cgroup Driver/{print $3}')
 cat <<EOF > /lib/systemd/system/frakti.service

hack/lib/hyper.sh

@@ -82,8 +82,8 @@ frakti::hyper::preinstall() {
     return 0
   fi
   sudo apt-get update -qq
-  sudo apt-get install -y wget autoconf automake pkg-config libdevmapper-dev libsqlite3-dev libvirt-dev libvirt-bin libaio1 libpixman-1-0 -qq
-  wget https://s3-us-west-1.amazonaws.com/hypercontainer-download/qemu-hyper/qemu-hyper_2.4.1-1_amd64.deb && sudo dpkg -i --force-all qemu-hyper_2.4.1-1_amd64.deb
+  sudo apt-get install -y qemu-system-x86 wget autoconf automake pkg-config libdevmapper-dev libsqlite3-dev libvirt-dev libvirt-bin libaio1 libpixman-1-0 -qq
+  # wget https://s3-us-west-1.amazonaws.com/hypercontainer-download/qemu-hyper/qemu-hyper_2.4.1-1_amd64.deb && sudo dpkg -i --force-all qemu-hyper_2.4.1-1_amd64.deb
 }
 
 frakti::hyper::export_related_path() {

pkg/docker/privilegedruntime.go

@@ -17,7 +17,6 @@ limitations under the License.
 package docker
 
 import (
-	"fmt"
 	"net/http"
 	"os"
 
@@ -31,13 +30,15 @@ import (
 	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/dockershim"
-	"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 )
 
 const (
+	// NOTE(harry): all consts defined here mean user configure of kubelet like NonMasqueradeCIDR, will be ignored.
+	// This can be fixed when dockershim become independent, then we can delete all these default values.
 	networkPluginName = "cni"
 	networkPluginMTU  = 1460
+	nonMasqueradeCIDR = "10.0.0.0/8"
 )
 
 type PrivilegedRuntime struct {
@@ -65,43 +66,37 @@ func NewPrivilegedRuntimeService(privilegedRuntimeEndpoint string, streamingConf
 	}
 
 	crOption := options.NewContainerRuntimeOptions()
-	dockerClient := libdocker.ConnectToDockerOrDie(
-		// privilegedRuntimeEndpoint defaults to kubeCfg.DockerEndpoint
-		privilegedRuntimeEndpoint,
-		kubeCfg.RuntimeRequestTimeout.Duration,
-		crOption.ImagePullProgressDeadline.Duration,
-	)
-	// TODO(resouer) is it fine to reuse the CNI plug-in?
+
+	dockerClientConfig := &dockershim.ClientConfig{
+		DockerEndpoint:            privilegedRuntimeEndpoint,
+		RuntimeRequestTimeout:     kubeCfg.RuntimeRequestTimeout.Duration,
+		ImagePullProgressDeadline: crOption.ImagePullProgressDeadline.Duration,
+	}
+
+	// NOTE(harry): pluginSettings should be arguments for dockershim, not part of kubelet.
+	// But standalone dockershim is not ready yet, so we use default values here.
 	pluginSettings := dockershim.NetworkPluginSettings{
 		HairpinMode:       kubeletconfiginternal.HairpinMode(kubeCfg.HairpinMode),
-		NonMasqueradeCIDR: kubeCfg.NonMasqueradeCIDR,
+		NonMasqueradeCIDR: nonMasqueradeCIDR,
 		PluginName:        networkPluginName,
 		PluginConfDir:     cniNetDir,
 		PluginBinDir:      cniPluginDir,
 		MTU:               networkPluginMTU,
 	}
 	var nl *kubelet.NoOpLegacyHost
 	pluginSettings.LegacyRuntimeHost = nl
-	// set cgroup driver to dockershim
-	dockerInfo, err := dockerClient.Info()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get info from docker: %v", err)
-	}
-	if len(dockerInfo.CgroupDriver) == 0 {
-		glog.Warningf("No cgroup driver is set in Docker, use frakti configuration: %q", cgroupDriver)
-	} else if dockerInfo.CgroupDriver != cgroupDriver {
-		return nil, fmt.Errorf("misconfiguration: frakti cgroup driver: %q is different from docker cgroup driver: %q", dockerInfo.CgroupDriver, cgroupDriver)
-	}
+
 	ds, err := dockershim.NewDockerService(
-		dockerClient,
+		dockerClientConfig,
 		crOption.PodSandboxImage,
 		streamingConfig,
 		&pluginSettings,
-		kubeCfg.RuntimeCgroups,
+		// RuntimeCgroups is optional, so we will not set it here.
+		"",
+		// If dockershim detected this cgroupDriver is different with dockerd, it will fail.
 		cgroupDriver,
-		crOption.DockerExecHandlerName,
 		privilegedRuntimeRootDir,
-		crOption.DockerDisableSharedPID,
+		true,
 	)
 	if err != nil {
 		return nil, err

pkg/hyper/network.go

@@ -24,7 +24,7 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/containernetworking/cni/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/golang/glog"
 	"github.com/vishvananda/netlink"
 )

pkg/hyper/sandbox.go

@@ -21,7 +21,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/containernetworking/cni/pkg/ns"
+	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/golang/glog"
 	"golang.org/x/sys/unix"
 

pkg/hyper/streaming_test.go

@@ -143,6 +143,7 @@ func TestRuntimeExec(t *testing.T) {
 	//Test Runtime Exec
 	rawContainerID := fmt.Sprintf("%s%s%d", containerId, "*", 0)
 	execRequest := &kubeapi.ExecRequest{
+		Stdout:      true,
 		ContainerId: rawContainerID,
 	}
 	execResponse, err := r.Exec(execRequest)
@@ -156,6 +157,7 @@ func TestRuntimeExec(t *testing.T) {
 	rawContainerID = fmt.Sprintf("%s%s%d", containerId, "*", 1)
 	attachRequest := &kubeapi.AttachRequest{
 		ContainerId: rawContainerID,
+		Stdout:      true,
 	}
 	attachResponse, err := r.Attach(attachRequest)
 	assert.NoError(t, err)

pkg/util/network/network.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"net"
+)
+
+// GetLocalIPAddress returns default local IP address of the host machine.
+func GetLocalIPAddress() (string, error) {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "", err
+	}
+	for _, addr := range addrs {
+		var ip net.IP
+		switch v := addr.(type) {
+		case *net.IPNet:
+			ip = v.IP
+		case *net.IPAddr:
+			ip = v.IP
+		}
+		if ip != nil && !ip.IsLoopback() && ip.To4() != nil {
+			return ip.String(), nil
+		}
+	}
+
+	return "", nil
+}