feat: Support uploading roles and permssions via xlsx files. (#1899)

* Support uploading roles and permissions via xlsx file.

* Template xlsx file for uploading users and permissions.

* reformat according to gofumpt.

* fix typo.

controllers/permission_upload.go

@@ -0,0 +1,50 @@
+// Copyright 2023 The Casdoor Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+	"fmt"
+
+	"github.com/casdoor/casdoor/object"
+	"github.com/casdoor/casdoor/util"
+)
+
+func (c *ApiController) UploadPermissions() {
+	userId := c.GetSessionUsername()
+	owner, user := util.GetOwnerAndNameFromId(userId)
+
+	file, header, err := c.Ctx.Request.FormFile("file")
+	if err != nil {
+		c.ResponseError(err.Error())
+		return
+	}
+
+	fileId := fmt.Sprintf("%s_%s_%s", owner, user, util.RemoveExt(header.Filename))
+
+	path := util.GetUploadXlsxPath(fileId)
+	util.EnsureFileFolderExists(path)
+	err = saveFile(path, &file)
+	if err != nil {
+		c.ResponseError(err.Error())
+		return
+	}
+
+	affected := object.UploadPermissions(owner, fileId)
+	if affected {
+		c.ResponseOk()
+	} else {
+		c.ResponseError(c.T("user_upload:Failed to import users"))
+	}
+}

controllers/role_upload.go

@@ -0,0 +1,50 @@
+// Copyright 2023 The Casdoor Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+	"fmt"
+
+	"github.com/casdoor/casdoor/object"
+	"github.com/casdoor/casdoor/util"
+)
+
+func (c *ApiController) UploadRoles() {
+	userId := c.GetSessionUsername()
+	owner, user := util.GetOwnerAndNameFromId(userId)
+
+	file, header, err := c.Ctx.Request.FormFile("file")
+	if err != nil {
+		c.ResponseError(err.Error())
+		return
+	}
+
+	fileId := fmt.Sprintf("%s_%s_%s", owner, user, util.RemoveExt(header.Filename))
+
+	path := util.GetUploadXlsxPath(fileId)
+	util.EnsureFileFolderExists(path)
+	err = saveFile(path, &file)
+	if err != nil {
+		c.ResponseError(err.Error())
+		return
+	}
+
+	affected := object.UploadRoles(owner, fileId)
+	if affected {
+		c.ResponseOk()
+	} else {
+		c.ResponseError(c.T("user_upload:Failed to import users"))
+	}
+}

object/permission.go

@@ -15,6 +15,9 @@
 package object
 
 import (
+	"strings"
+
+	"github.com/casdoor/casdoor/conf"
 	"github.com/casdoor/casdoor/util"
 	"github.com/xorm-io/core"
 )
@@ -188,6 +191,54 @@ func AddPermission(permission *Permission) bool {
 	return affected != 0
 }
 
+func AddPermissions(permissions []*Permission) bool {
+	if len(permissions) == 0 {
+		return false
+	}
+
+	affected, err := adapter.Engine.Insert(permissions)
+	if err != nil {
+		if !strings.Contains(err.Error(), "Duplicate entry") {
+			panic(err)
+		}
+	}
+
+	for _, permission := range permissions {
+		// add using for loop
+		if affected != 0 {
+			addGroupingPolicies(permission)
+			addPolicies(permission)
+		}
+	}
+	return affected != 0
+}
+
+func AddPermissionsInBatch(permissions []*Permission) bool {
+	batchSize := conf.GetConfigBatchSize()
+
+	if len(permissions) == 0 {
+		return false
+	}
+
+	affected := false
+	for i := 0; i < (len(permissions)-1)/batchSize+1; i++ {
+		start := i * batchSize
+		end := (i + 1) * batchSize
+		if end > len(permissions) {
+			end = len(permissions)
+		}
+
+		tmp := permissions[start:end]
+		// TODO: save to log instead of standard output
+		// fmt.Printf("Add Permissions: [%d - %d].\n", start, end)
+		if AddPermissions(tmp) {
+			affected = true
+		}
+	}
+
+	return affected
+}
+
 func DeletePermission(permission *Permission) bool {
 	affected, err := adapter.Engine.ID(core.PK{permission.Owner, permission.Name}).Delete(&Permission{})
 	if err != nil {

object/permission_upload.go

@@ -0,0 +1,77 @@
+// Copyright 2023 The Casdoor Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package object
+
+import (
+	"github.com/casdoor/casdoor/xlsx"
+)
+
+func getPermissionMap(owner string) map[string]*Permission {
+	m := map[string]*Permission{}
+
+	permissions := GetPermissions(owner)
+	for _, permission := range permissions {
+		m[permission.GetId()] = permission
+	}
+
+	return m
+}
+
+func UploadPermissions(owner string, fileId string) bool {
+	table := xlsx.ReadXlsxFile(fileId)
+
+	oldUserMap := getPermissionMap(owner)
+	newPermissions := []*Permission{}
+	for index, line := range table {
+		if index == 0 || parseLineItem(&line, 0) == "" {
+			continue
+		}
+
+		permission := &Permission{
+			Owner:       parseLineItem(&line, 0),
+			Name:        parseLineItem(&line, 1),
+			CreatedTime: parseLineItem(&line, 2),
+			DisplayName: parseLineItem(&line, 3),
+
+			Users:   parseListItem(&line, 4),
+			Roles:   parseListItem(&line, 5),
+			Domains: parseListItem(&line, 6),
+
+			Model:        parseLineItem(&line, 7),
+			Adapter:      parseLineItem(&line, 8),
+			ResourceType: parseLineItem(&line, 9),
+
+			Resources: parseListItem(&line, 10),
+			Actions:   parseListItem(&line, 11),
+
+			Effect:    parseLineItem(&line, 12),
+			IsEnabled: parseLineItemBool(&line, 13),
+
+			Submitter:   parseLineItem(&line, 14),
+			Approver:    parseLineItem(&line, 15),
+			ApproveTime: parseLineItem(&line, 16),
+			State:       parseLineItem(&line, 17),
+		}
+
+		if _, ok := oldUserMap[permission.GetId()]; !ok {
+			newPermissions = append(newPermissions, permission)
+		}
+	}
+
+	if len(newPermissions) == 0 {
+		return false
+	}
+	return AddPermissionsInBatch(newPermissions)
+}

object/role.go

@@ -16,6 +16,9 @@ package object
 
 import (
 	"fmt"
+	"strings"
+
+	"github.com/casdoor/casdoor/conf"
 
 	"github.com/casdoor/casdoor/util"
 	"github.com/xorm-io/core"
@@ -160,6 +163,45 @@ func AddRole(role *Role) bool {
 	return affected != 0
 }
 
+func AddRoles(roles []*Role) bool {
+	if len(roles) == 0 {
+		return false
+	}
+	affected, err := adapter.Engine.Insert(roles)
+	if err != nil {
+		if !strings.Contains(err.Error(), "Duplicate entry") {
+			panic(err)
+		}
+	}
+	return affected != 0
+}
+
+func AddRolesInBatch(roles []*Role) bool {
+	batchSize := conf.GetConfigBatchSize()
+
+	if len(roles) == 0 {
+		return false
+	}
+
+	affected := false
+	for i := 0; i < (len(roles)-1)/batchSize+1; i++ {
+		start := i * batchSize
+		end := (i + 1) * batchSize
+		if end > len(roles) {
+			end = len(roles)
+		}
+
+		tmp := roles[start:end]
+		// TODO: save to log instead of standard output
+		// fmt.Printf("Add users: [%d - %d].\n", start, end)
+		if AddRoles(tmp) {
+			affected = true
+		}
+	}
+
+	return affected
+}
+
 func DeleteRole(role *Role) bool {
 	roleId := role.GetId()
 	permissions := GetPermissionsByRole(roleId)

object/role_upload.go

@@ -0,0 +1,63 @@
+// Copyright 2023 The Casdoor Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package object
+
+import (
+	"github.com/casdoor/casdoor/xlsx"
+)
+
+func getRoleMap(owner string) map[string]*Role {
+	m := map[string]*Role{}
+
+	roles := GetRoles(owner)
+	for _, role := range roles {
+		m[role.GetId()] = role
+	}
+
+	return m
+}
+
+func UploadRoles(owner string, fileId string) bool {
+	table := xlsx.ReadXlsxFile(fileId)
+
+	oldUserMap := getRoleMap(owner)
+	newRoles := []*Role{}
+	for index, line := range table {
+		if index == 0 || parseLineItem(&line, 0) == "" {
+			continue
+		}
+
+		role := &Role{
+			Owner:       parseLineItem(&line, 0),
+			Name:        parseLineItem(&line, 1),
+			CreatedTime: parseLineItem(&line, 2),
+			DisplayName: parseLineItem(&line, 3),
+
+			Users:     parseListItem(&line, 4),
+			Roles:     parseListItem(&line, 5),
+			Domains:   parseListItem(&line, 6),
+			IsEnabled: parseLineItemBool(&line, 7),
+		}
+
+		if _, ok := oldUserMap[role.GetId()]; !ok {
+			newRoles = append(newRoles, role)
+		}
+	}
+
+	if len(newRoles) == 0 {
+		return false
+	}
+	return AddRolesInBatch(newRoles)
+}

object/user_upload.go

@@ -15,6 +15,9 @@
 package object
 
 import (
+	"sort"
+	"strings"
+
 	"github.com/casdoor/casdoor/util"
 	"github.com/casdoor/casdoor/xlsx"
 )
@@ -47,6 +50,26 @@ func parseLineItemBool(line *[]string, i int) bool {
 	return parseLineItemInt(line, i) != 0
 }
 
+func parseListItem(lines *[]string, i int) []string {
+	if i >= len(*lines) {
+		return nil
+	}
+	line := (*lines)[i]
+	items := strings.Split(line, ";")
+	trimmedItems := make([]string, 0, len(items))
+
+	for _, item := range items {
+		trimmedItem := strings.TrimSpace(item)
+		if trimmedItem != "" {
+			trimmedItems = append(trimmedItems, trimmedItem)
+		}
+	}
+
+	sort.Strings(trimmedItems)
+
+	return trimmedItems
+}
+
 func UploadUsers(owner string, fileId string) bool {
 	table := xlsx.ReadXlsxFile(fileId)
 

routers/router.go

@@ -82,6 +82,7 @@ func initAPI() {
 	beego.Router("/api/update-role", &controllers.ApiController{}, "POST:UpdateRole")
 	beego.Router("/api/add-role", &controllers.ApiController{}, "POST:AddRole")
 	beego.Router("/api/delete-role", &controllers.ApiController{}, "POST:DeleteRole")
+	beego.Router("/api/upload-roles", &controllers.ApiController{}, "POST:UploadRoles")
 
 	beego.Router("/api/get-permissions", &controllers.ApiController{}, "GET:GetPermissions")
 	beego.Router("/api/get-permissions-by-submitter", &controllers.ApiController{}, "GET:GetPermissionsBySubmitter")
@@ -90,6 +91,7 @@ func initAPI() {
 	beego.Router("/api/update-permission", &controllers.ApiController{}, "POST:UpdatePermission")
 	beego.Router("/api/add-permission", &controllers.ApiController{}, "POST:AddPermission")
 	beego.Router("/api/delete-permission", &controllers.ApiController{}, "POST:DeletePermission")
+	beego.Router("/api/upload-permissions", &controllers.ApiController{}, "POST:UploadPermissions")
 
 	beego.Router("/api/enforce", &controllers.ApiController{}, "POST:Enforce")
 	beego.Router("/api/batch-enforce", &controllers.ApiController{}, "POST:BatchEnforce")

web/src/PermissionListPage.js

@@ -14,13 +14,14 @@
 
 import React from "react";
 import {Link} from "react-router-dom";
-import {Button, Switch, Table} from "antd";
+import {Button, Switch, Table, Upload} from "antd";
 import moment from "moment";
 import * as Setting from "./Setting";
 import * as PermissionBackend from "./backend/PermissionBackend";
 import i18next from "i18next";
 import BaseListPage from "./BaseListPage";
 import PopconfirmModal from "./common/modal/PopconfirmModal";
+import {UploadOutlined} from "@ant-design/icons";
 
 class PermissionListPage extends BaseListPage {
   newPermission() {
@@ -79,6 +80,40 @@ class PermissionListPage extends BaseListPage {
       });
   }
 
+  uploadPermissionFile(info) {
+    const {status, response: res} = info.file;
+    if (status === "done") {
+      if (res.status === "ok") {
+        Setting.showMessage("success", "Users uploaded successfully, refreshing the page");
+
+        const {pagination} = this.state;
+        this.fetch({pagination});
+      } else {
+        Setting.showMessage("error", `Users failed to upload: ${res.msg}`);
+      }
+    } else if (status === "error") {
+      Setting.showMessage("error", "File failed to upload");
+    }
+  }
+  renderPermissionUpload() {
+    const props = {
+      name: "file",
+      accept: ".xlsx",
+      method: "post",
+      action: `${Setting.ServerUrl}/api/upload-permissions`,
+      withCredentials: true,
+      onChange: (info) => {
+        this.uploadPermissionFile(info);
+      },
+    };
+
+    return (
+      <Upload {...props}>
+        <Button type="primary" size="small">
+          <UploadOutlined /> {i18next.t("user:Upload (.xlsx)")}
+        </Button></Upload>
+    );
+  }
   renderTable(permissions) {
     const columns = [
       // https://github.com/ant-design/ant-design/issues/22184
@@ -325,7 +360,10 @@ class PermissionListPage extends BaseListPage {
           title={() => (
             <div>
               {i18next.t("general:Permissions")}&nbsp;&nbsp;&nbsp;&nbsp;
-              <Button type="primary" size="small" onClick={this.addPermission.bind(this)}>{i18next.t("general:Add")}</Button>
+              <Button style={{marginRight: "5px"}} type="primary" size="small" onClick={this.addPermission.bind(this)}>{i18next.t("general:Add")}</Button>
+              {
+                this.renderPermissionUpload()
+              }
             </div>
           )}
           loading={this.state.loading}

web/src/RoleListPage.js

@@ -14,13 +14,14 @@
 
 import React from "react";
 import {Link} from "react-router-dom";
-import {Button, Switch, Table} from "antd";
+import {Button, Switch, Table, Upload} from "antd";
 import moment from "moment";
 import * as Setting from "./Setting";
 import * as RoleBackend from "./backend/RoleBackend";
 import i18next from "i18next";
 import BaseListPage from "./BaseListPage";
 import PopconfirmModal from "./common/modal/PopconfirmModal";
+import {UploadOutlined} from "@ant-design/icons";
 
 class RoleListPage extends BaseListPage {
   newRole() {
@@ -71,6 +72,42 @@ class RoleListPage extends BaseListPage {
       });
   }
 
+  uploadRoleFile(info) {
+    const {status, response: res} = info.file;
+    if (status === "done") {
+      if (res.status === "ok") {
+        Setting.showMessage("success", "Users uploaded successfully, refreshing the page");
+
+        const {pagination} = this.state;
+        this.fetch({pagination});
+      } else {
+        Setting.showMessage("error", `Users failed to upload: ${res.msg}`);
+      }
+    } else if (status === "error") {
+      Setting.showMessage("error", "File failed to upload");
+    }
+  }
+
+  renderRoleUpload() {
+    const props = {
+      name: "file",
+      accept: ".xlsx",
+      method: "post",
+      action: `${Setting.ServerUrl}/api/upload-roles`,
+      withCredentials: true,
+      onChange: (info) => {
+        this.uploadRoleFile(info);
+      },
+    };
+
+    return (
+      <Upload {...props}>
+        <Button type="primary" size="small">
+          <UploadOutlined /> {i18next.t("user:Upload (.xlsx)")}
+        </Button>
+      </Upload>
+    );
+  }
   renderTable(roles) {
     const columns = [
       {
@@ -200,7 +237,10 @@ class RoleListPage extends BaseListPage {
           title={() => (
             <div>
               {i18next.t("general:Roles")}&nbsp;&nbsp;&nbsp;&nbsp;
-              <Button type="primary" size="small" onClick={this.addRole.bind(this)}>{i18next.t("general:Add")}</Button>
+              <Button style={{marginRight: "5px"}} type="primary" size="small" onClick={this.addRole.bind(this)}>{i18next.t("general:Add")}</Button>
+              {
+                this.renderRoleUpload()
+              }
             </div>
           )}
           loading={this.state.loading}