From 93da329b560f233711b6ff3a3c7ae9ae90d1b0a5 Mon Sep 17 00:00:00 2001 From: "zongyi.chen" Date: Wed, 30 Nov 2022 11:19:17 +0800 Subject: [PATCH 1/6] Added a demo for calculating the median of two parties. --- .../paddle_fl/mpc/tests/demo/mid/mid2/data.py | 9 ++ .../mpc/tests/demo/mid/mid2/flush.py | 4 + .../mpc/tests/demo/mid/mid2/mid2_C0.py | 86 +++++++++++ .../mpc/tests/demo/mid/mid2/mid2_C1.py | 86 +++++++++++ .../mpc/tests/demo/mid/mid2/mid2_C2.py | 86 +++++++++++ .../mpc/tests/demo/mid/mid2/mid2_share.py | 46 ++++++ .../mpc/tests/demo/mid/mid2/reconstruct.py | 15 ++ .../paddle_fl/mpc/tests/demo/mid/mid2/run.sh | 5 + .../mpc/tests/demo/mid/mid2/test_op_base.py | 140 ++++++++++++++++++ 9 files changed, 477 insertions(+) create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/data.py create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/flush.py create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C0.py create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C1.py create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C2.py create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_share.py create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/reconstruct.py create mode 100755 python/paddle_fl/mpc/tests/demo/mid/mid2/run.sh create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/test_op_base.py diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/data.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/data.py new file mode 100644 index 0000000..706d1fc --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/data.py @@ -0,0 +1,9 @@ +import random + +file_handle1=open('Input-P1.list',mode='w') +for i in range(20): + file_handle1.write(str(float(random.randint(1,10000)+round(random.random(),18))) + '\n') + +file_handle1=open('Input-P2.list',mode='w') +for i in range(20): + file_handle1.write(str(float(random.randint(5000,9000)+round(random.random(),18))) + '\n') \ No newline at end of file diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/flush.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/flush.py new file mode 100644 index 0000000..b077063 --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/flush.py @@ -0,0 +1,4 @@ +import redis + +r = redis.Redis(host="127.0.0.1", port=6379) +r.flushall() diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C0.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C0.py new file mode 100644 index 0000000..b3c5ed3 --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C0.py @@ -0,0 +1,86 @@ +import unittest +from multiprocessing import Manager +import numpy as np +import paddle.fluid as fluid +import paddle_fl.mpc as pfl_mpc +import test_op_base +from paddle_fl.mpc.data_utils.data_utils import get_datautils + +aby3 = get_datautils('aby3') + +class Solution(test_op_base.TestOpBase): + def findMedianSortedArrays(self, **kwargs): + def getKthElement(k): + index1, index2 = 0, 0 + op_sub = pfl_mpc.layers.elementwise_sub(x=x, y=y) + op_gt = pfl_mpc.layers.greater_than(x=x, y=zero) + while True: + if index1 == d_1_length: + return d_2[:,index2 + k - 1:index2 + k],index1, index2 + k,1 + if index2 == d_2_length: + return d_1[:,index1 + k - 1:index1 + k],index1 + k, index2,0 + if k == 1: + d_tmp = exe.run(feed={'x': d_1[:,index1:index1+1], 'y': d_2[:,index2:index2+1],'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y':d_2[:,index2:index2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 1: + return d_2[:,index2:index2+1],index1, index2 + 1,1 + else: + return d_1[:,index1:index1+1],index1 + 1, index2,0 + + newIndex1 = min(index1 + k // 2 - 1, d_1_length - 1) + newIndex2 = min(index2 + k // 2 - 1, d_2_length - 1) + d_tmp = exe.run(feed={'x': d_1[:,newIndex1:newIndex1+1], 'y': d_2[:,newIndex2:newIndex2+1], 'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y': d_2[:,newIndex2:newIndex2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 0: + k -= newIndex1 - index1 + 1 + index1 = newIndex1 + 1 + else: + k -= newIndex2 - index2 + 1 + index2 = newIndex2 + 1 + + def getNextElement(index1,index2,num): + op_sub = pfl_mpc.layers.elementwise_sub(x=x, y=y) + op_gt = pfl_mpc.layers.greater_than(x=x, y=zero) + if(num == 0 and index1 == d_1_length): + return d_2[:,index2:index2+1] + if(num == 1 and index2 == d_2_length): + return d_1[:,index1:index1+1] + else: + d_tmp = exe.run(feed={'x': d_1[:,index1:index1+1], 'y': d_2[:,index2:index2+1],'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y':d_2[:,index2:index2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 1: + return d_2[:,index2:index2+1] + else: + return d_1[:,index1:index1+1] + + role = 0 + d_1 = np.load('data_C0_P1.npy',allow_pickle=True) + d_2 = np.load('data_C0_P2.npy',allow_pickle=True) + d_zero = np.full((1), fill_value=0).astype('float32') + pfl_mpc.init("aby3", role, "localhost", self.server, int(self.port)) + x = pfl_mpc.data(name='x', shape=[1], dtype='int64') + y = pfl_mpc.data(name='y', shape=[1], dtype='int64') + zero = fluid.data(name='zero', shape=[1], dtype='float32') + op_add = pfl_mpc.layers.elementwise_add(x=x, y=y) + math_mul = pfl_mpc.layers.elementwise_mul(x, y) + exe = fluid.Executor(place=fluid.CPUPlace()) + + d_1_length, d_2_length = d_1.shape[1], d_2.shape[1] + totalLength = d_1_length + d_2_length + if totalLength % 2 == 1: + results = getKthElement((totalLength + 1) // 2) + np.save('result_C0.npy', results[0]) + else: + op_add = pfl_mpc.layers.elementwise_add(x=x, y=y) + mid_pre = getKthElement(totalLength // 2) + mid_post = getNextElement(mid_pre[1],mid_pre[2],mid_pre[3]) + d_tmp = np.load('data_C0_tmp.npy',allow_pickle=True) + tmp = exe.run(feed={'x': mid_pre[0], 'y': mid_post, 'zero': d_zero}, fetch_list=[op_add]) + results = exe.run(feed={'x': tmp[0],'y':d_tmp,'zero': d_zero}, fetch_list=[math_mul]) + np.save('result_C0.npy', results[0]) + + def test_mid2_C0(self): + ret = self.multi_party_run0(target=self.findMedianSortedArrays) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C1.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C1.py new file mode 100644 index 0000000..83f0bf2 --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C1.py @@ -0,0 +1,86 @@ +import unittest +from multiprocessing import Manager +import numpy as np +import paddle.fluid as fluid +import paddle_fl.mpc as pfl_mpc +import test_op_base +from paddle_fl.mpc.data_utils.data_utils import get_datautils + +aby3 = get_datautils('aby3') + +class Solution(test_op_base.TestOpBase): + def findMedianSortedArrays(self, **kwargs): + def getKthElement(k): + index1, index2 = 0, 0 + op_sub = pfl_mpc.layers.elementwise_sub(x=x, y=y) + op_gt = pfl_mpc.layers.greater_than(x=x, y=zero) + while True: + if index1 == d_1_length: + return d_2[:,index2 + k - 1:index2 + k],index1, index2 + k,1 + if index2 == d_2_length: + return d_1[:,index1 + k - 1:index1 + k],index1 + k, index2,0 + if k == 1: + d_tmp = exe.run(feed={'x': d_1[:,index1:index1+1], 'y': d_2[:,index2:index2+1],'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y':d_2[:,index2:index2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 1: + return d_2[:,index2:index2+1],index1, index2 + 1,1 + else: + return d_1[:,index1:index1+1],index1 + 1, index2,0 + + newIndex1 = min(index1 + k // 2 - 1, d_1_length - 1) + newIndex2 = min(index2 + k // 2 - 1, d_2_length - 1) + d_tmp = exe.run(feed={'x': d_1[:,newIndex1:newIndex1+1], 'y': d_2[:,newIndex2:newIndex2+1], 'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y': d_2[:,newIndex2:newIndex2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 0: + k -= newIndex1 - index1 + 1 + index1 = newIndex1 + 1 + else: + k -= newIndex2 - index2 + 1 + index2 = newIndex2 + 1 + + def getNextElement(index1,index2,num): + op_sub = pfl_mpc.layers.elementwise_sub(x=x, y=y) + op_gt = pfl_mpc.layers.greater_than(x=x, y=zero) + if(num == 0 and index1 == d_1_length): + return d_2[:,index2:index2+1] + if(num == 1 and index2 == d_2_length): + return d_1[:,index1:index1+1] + else: + d_tmp = exe.run(feed={'x': d_1[:,index1:index1+1], 'y': d_2[:,index2:index2+1],'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y':d_2[:,index2:index2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 1: + return d_2[:,index2:index2+1] + else: + return d_1[:,index1:index1+1] + + role = 1 + d_1 = np.load('data_C1_P1.npy',allow_pickle=True) + d_2 = np.load('data_C1_P2.npy',allow_pickle=True) + d_zero = np.full((1), fill_value=0).astype('float32') + pfl_mpc.init("aby3", role, "localhost", self.server, int(self.port)) + x = pfl_mpc.data(name='x', shape=[1], dtype='int64') + y = pfl_mpc.data(name='y', shape=[1], dtype='int64') + zero = fluid.data(name='zero', shape=[1], dtype='float32') + op_add = pfl_mpc.layers.elementwise_add(x=x, y=y) + math_mul = pfl_mpc.layers.elementwise_mul(x, y) + exe = fluid.Executor(place=fluid.CPUPlace()) + + d_1_length, d_2_length = d_1.shape[1], d_2.shape[1] + totalLength = d_1_length + d_2_length + if totalLength % 2 == 1: + results = getKthElement((totalLength + 1) // 2) + np.save('result_C1.npy', results[0]) + else: + op_add = pfl_mpc.layers.elementwise_add(x=x, y=y) + mid_pre = getKthElement(totalLength // 2) + mid_post = getNextElement(mid_pre[1],mid_pre[2],mid_pre[3]) + d_tmp = np.load('data_C1_tmp.npy',allow_pickle=True) + tmp = exe.run(feed={'x': mid_pre[0], 'y': mid_post, 'zero': d_zero}, fetch_list=[op_add]) + results = exe.run(feed={'x': tmp[0],'y':d_tmp,'zero': d_zero}, fetch_list=[math_mul]) + np.save('result_C1.npy', results[0]) + + def test_mid2_C1(self): + ret = self.multi_party_run1(target=self.findMedianSortedArrays) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C2.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C2.py new file mode 100644 index 0000000..eccc6ee --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_C2.py @@ -0,0 +1,86 @@ +import unittest +from multiprocessing import Manager +import numpy as np +import paddle.fluid as fluid +import paddle_fl.mpc as pfl_mpc +import test_op_base +from paddle_fl.mpc.data_utils.data_utils import get_datautils + +aby3 = get_datautils('aby3') + +class Solution(test_op_base.TestOpBase): + def findMedianSortedArrays(self, **kwargs): + def getKthElement(k): + index1, index2 = 0, 0 + op_sub = pfl_mpc.layers.elementwise_sub(x=x, y=y) + op_gt = pfl_mpc.layers.greater_than(x=x, y=zero) + while True: + if index1 == d_1_length: + return d_2[:,index2 + k - 1:index2 + k],index1, index2 + k,1 + if index2 == d_2_length: + return d_1[:,index1 + k - 1:index1 + k],index1 + k, index2,0 + if k == 1: + d_tmp = exe.run(feed={'x': d_1[:,index1:index1+1], 'y': d_2[:,index2:index2+1],'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y':d_2[:,index2:index2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 1: + return d_2[:,index2:index2+1],index1, index2 + 1,1 + else: + return d_1[:,index1:index1+1],index1 + 1, index2,0 + + newIndex1 = min(index1 + k // 2 - 1, d_1_length - 1) + newIndex2 = min(index2 + k // 2 - 1, d_2_length - 1) + d_tmp = exe.run(feed={'x': d_1[:,newIndex1:newIndex1+1], 'y': d_2[:,newIndex2:newIndex2+1], 'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y': d_2[:,newIndex2:newIndex2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 0: + k -= newIndex1 - index1 + 1 + index1 = newIndex1 + 1 + else: + k -= newIndex2 - index2 + 1 + index2 = newIndex2 + 1 + + def getNextElement(index1,index2,num): + op_sub = pfl_mpc.layers.elementwise_sub(x=x, y=y) + op_gt = pfl_mpc.layers.greater_than(x=x, y=zero) + if(num == 0 and index1 == d_1_length): + return d_2[:,index2:index2+1] + if(num == 1 and index2 == d_2_length): + return d_1[:,index1:index1+1] + else: + d_tmp = exe.run(feed={'x': d_1[:,index1:index1+1], 'y': d_2[:,index2:index2+1],'zero': d_zero}, fetch_list=[op_sub]) + results = exe.run(feed={'x': d_tmp[0],'y':d_2[:,index2:index2+1], 'zero': d_zero}, fetch_list=[op_gt]) + if results[0] == 1: + return d_2[:,index2:index2+1] + else: + return d_1[:,index1:index1+1] + + role = 2 + d_1 = np.load('data_C2_P1.npy',allow_pickle=True) + d_2 = np.load('data_C2_P2.npy',allow_pickle=True) + d_zero = np.full((1), fill_value=0).astype('float32') + pfl_mpc.init("aby3", role, "localhost", self.server, int(self.port)) + x = pfl_mpc.data(name='x', shape=[1], dtype='int64') + y = pfl_mpc.data(name='y', shape=[1], dtype='int64') + zero = fluid.data(name='zero', shape=[1], dtype='float32') + op_add = pfl_mpc.layers.elementwise_add(x=x, y=y) + math_mul = pfl_mpc.layers.elementwise_mul(x, y) + exe = fluid.Executor(place=fluid.CPUPlace()) + + d_1_length, d_2_length = d_1.shape[1], d_2.shape[1] + totalLength = d_1_length + d_2_length + if totalLength % 2 == 1: + results = getKthElement((totalLength + 1) // 2) + np.save('result_C2.npy', results[0]) + else: + op_add = pfl_mpc.layers.elementwise_add(x=x, y=y) + mid_pre = getKthElement(totalLength // 2) + mid_post = getNextElement(mid_pre[1],mid_pre[2],mid_pre[3]) + d_tmp = np.load('data_C2_tmp.npy',allow_pickle=True) + tmp = exe.run(feed={'x': mid_pre[0], 'y': mid_post, 'zero': d_zero}, fetch_list=[op_add]) + results = exe.run(feed={'x': tmp[0],'y':d_tmp,'zero': d_zero}, fetch_list=[math_mul]) + np.save('result_C2.npy', results[0]) + + def test_mid2_C2(self): + ret = self.multi_party_run2(target=self.findMedianSortedArrays) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_share.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_share.py new file mode 100644 index 0000000..59aeaae --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/mid2_share.py @@ -0,0 +1,46 @@ +import numpy as np +from paddle_fl.mpc.data_utils.data_utils import get_datautils + +aby3 = get_datautils('aby3') + +if __name__ == '__main__': + with open(r'Input-P1.list', 'r') as file: + content_list1 = file.readlines() + contentall_1 = np.array([float(x) for x in content_list1]) + with open(r'Input-P2.list', 'r') as file: + content_list2 = file.readlines() + contentall_2 = np.array([float(x) for x in content_list2]) + + data_1 = np.sort(contentall_1) + data_2 = np.sort(contentall_2) + print(data_1) + print(data_2) + len_1, len_2 = data_1.shape[0], data_2.shape[0] + total = len_1 + len_2 + + if total % 2 == 1: + total_data = np.append(data_1,data_2) + print(np.sort(total_data)[total // 2:total // 2 + 1]) + print(total_data) + else: + total_data = np.append(data_1,data_2) + print((np.sort(total_data)[total // 2 - 1:total // 2] + np.sort(total_data)[total // 2:total // 2 + 1]) * 0.5) + print(np.sort(total_data)) + data_tmp = np.array([0.5]) + data_tmp_shares = aby3.make_shares(data_tmp) + data_tmp_all3shares = np.array([aby3.get_shares(data_tmp_shares, i) for i in range(3)]) + np.save('data_C0_tmp.npy', data_tmp_all3shares[0]) + np.save('data_C1_tmp.npy', data_tmp_all3shares[1]) + np.save('data_C2_tmp.npy', data_tmp_all3shares[2]) + data_1_shares = aby3.make_shares(data_1) + data_2_shares = aby3.make_shares(data_2) + data_1_all3shares = np.array([aby3.get_shares(data_1_shares, i) for i in range(3)]) + data_2_all3shares = np.array([aby3.get_shares(data_2_shares, i) for i in range(3)]) + #print(data_1_all3shares[0].shape) + np.save('data_C0_P1.npy', data_1_all3shares[0]) + np.save('data_C0_P2.npy', data_2_all3shares[0]) + np.save('data_C1_P1.npy', data_1_all3shares[1]) + np.save('data_C1_P2.npy', data_2_all3shares[1]) + np.save('data_C2_P1.npy', data_1_all3shares[2]) + np.save('data_C2_P2.npy', data_2_all3shares[2]) + diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/reconstruct.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/reconstruct.py new file mode 100644 index 0000000..6b284e9 --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/reconstruct.py @@ -0,0 +1,15 @@ +import numpy as np +from paddle_fl.mpc.data_utils.data_utils import get_datautils + +aby3 = get_datautils('aby3') + +if __name__ == '__main__': + return_results = list() + result_C0 = np.load('result_C0.npy', allow_pickle=True) + result_C1 = np.load('result_C1.npy',allow_pickle=True) + result_C2 = np.load('result_C2.npy',allow_pickle=True) + return_results.append(result_C0) + return_results.append(result_C1) + return_results.append(result_C2) + revealed = aby3.reconstruct(np.array(return_results)) + print(revealed) \ No newline at end of file diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/run.sh b/python/paddle_fl/mpc/tests/demo/mid/mid2/run.sh new file mode 100755 index 0000000..b9c975e --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/run.sh @@ -0,0 +1,5 @@ +export TEST_REDIS_IP="127.0.0.1" +export TEST_REDIS_PORT="6379" +nohup python3 mid2_C0.py >> ./logs/console0.log 2>>./logs/error0.log & +nohup python3 mid2_C1.py >> ./logs/console1.log 2>>./logs/error1.log & +nohup python3 mid2_C2.py >> ./logs/console2.log 2>>./logs/error2.log & diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/test_op_base.py b/python/paddle_fl/mpc/tests/demo/mid/mid2/test_op_base.py new file mode 100644 index 0000000..aad2192 --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/test_op_base.py @@ -0,0 +1,140 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Set base config for op unit tests. +""" +from multiprocessing import Pipe, Process +import os +import traceback +import unittest + +import redis + + +class Aby3Process(Process): + """ + Extends from Process, evaluate the computation party in aby3. + """ + def __init__(self, *args, **kwargs): + Process.__init__(self, *args, **kwargs) + self._pconn, self._cconn = Pipe() + self._exception = None + + def run(self): + """ + Override. Send any exceptions raised in + subprocess to main process. + """ + try: + Process.run(self) + self._cconn.send(None) + except Exception as e: + tb = traceback.format_exc() + self._cconn.send((e, tb)) + + @property + def exception(self): + """ + Get exception. + """ + if self._pconn.poll(): + self._exception = self._pconn.recv() + return self._exception + + +class TestOpBase(unittest.TestCase): + def __init__(self, methodName='runTest'): + super(TestOpBase, self).__init__(methodName) + # set redis server and port + self.server = os.environ['TEST_REDIS_IP'] + self.port = os.environ['TEST_REDIS_PORT'] + self.party_num = 3 + + def setUp(self): + """ + Connect redis and delete all keys in all databases on the current host. + :return: + """ + r = redis.Redis(host=self.server, port=int(self.port)) + #r.flushall() + + def multi_party_run(self, **kwargs): + """ + Run 3 parties with target function or other additional arguments. + :param kwargs: + :return: + """ + target = kwargs['target'] + + parties = [] + for role in range(self.party_num): + kwargs.update({'role': role}) + parties.append(Aby3Process(target=target, kwargs=kwargs)) + parties[-1].start() + for party in parties: + party.join() + if party.exception: + return party.exception + return (True,) + + def multi_party_run0(self, **kwargs): + """ + Run 3 parties with target function or other additional arguments. + :param kwargs: + :return: + """ + target = kwargs['target'] + + parties = [] + kwargs.update({'role': 0}) + parties.append(Aby3Process(target=target, kwargs=kwargs)) + parties[-1].start() + parties[-1].join() + if parties[-1].exception: + return parties[-1].exception + return (True,) + + def multi_party_run1(self, **kwargs): + """ + Run 3 parties with target function or other additional arguments. + :param kwargs: + :return: + """ + target = kwargs['target'] + + parties = [] + kwargs.update({'role': 1}) + parties.append(Aby3Process(target=target, kwargs=kwargs)) + parties[-1].start() + parties[-1].join() + if parties[-1].exception: + return parties[-1].exception + return (True,) + + def multi_party_run2(self, **kwargs): + """ + Run 3 parties with target function or other additional arguments. + :param kwargs: + :return: + """ + target = kwargs['target'] + + parties = [] + kwargs.update({'role': 2}) + parties.append(Aby3Process(target=target, kwargs=kwargs)) + parties[-1].start() + parties[-1].join() + if parties[-1].exception: + return parties[-1].exception + return (True,) \ No newline at end of file -- Gitee From fa4135abde8c4b5913436e7670992ffd250f0db8 Mon Sep 17 00:00:00 2001 From: "zongyi.chen" Date: Wed, 30 Nov 2022 11:25:59 +0800 Subject: [PATCH 2/6] Added a demo for calculating the median of two parties. --- .../mpc/tests/demo/mid/mid2/logs/console0.log | 0 .../mpc/tests/demo/mid/mid2/logs/console1.log | 0 .../mpc/tests/demo/mid/mid2/logs/console2.log | 0 .../mpc/tests/demo/mid/mid2/logs/error0.log | 12 ++++++++++++ .../mpc/tests/demo/mid/mid2/logs/error1.log | 12 ++++++++++++ .../mpc/tests/demo/mid/mid2/logs/error2.log | 12 ++++++++++++ 6 files changed, 36 insertions(+) create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console0.log create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console1.log create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console2.log create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error0.log create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error1.log create mode 100644 python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error2.log diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console0.log b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console0.log new file mode 100644 index 0000000..e69de29 diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console1.log b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console1.log new file mode 100644 index 0000000..e69de29 diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console2.log b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/console2.log new file mode 100644 index 0000000..e69de29 diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error0.log b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error0.log new file mode 100644 index 0000000..04bf04c --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error0.log @@ -0,0 +1,12 @@ +/usr/local/python/lib/python3.8/site-packages/requests/__init__.py:102: RequestsDependencyWarning: urllib3 (1.26.6) or chardet (5.0.0)/charset_normalizer (2.0.1) doesn't match a supported version! + warnings.warn("urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " +/usr/local/python/lib/python3.8/site-packages/paddle/fluid/framework.py:2138: DeprecationWarning: an integer is required (got type paddle.fluid.core_avx.op_proto_and_checker_maker.OpRole). Implicit conversion to integers using __int__ is deprecated, and may be removed in a future version of Python. + self.desc._set_attr(name, val) +/usr/local/python/lib/python3.8/site-packages/paddle/fluid/framework.py:541: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here. +Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations + elif dtype == np.bool: +. +---------------------------------------------------------------------- +Ran 1 test in 0.282s + +OK diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error1.log b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error1.log new file mode 100644 index 0000000..482a951 --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error1.log @@ -0,0 +1,12 @@ +/usr/local/python/lib/python3.8/site-packages/requests/__init__.py:102: RequestsDependencyWarning: urllib3 (1.26.6) or chardet (5.0.0)/charset_normalizer (2.0.1) doesn't match a supported version! + warnings.warn("urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " +/usr/local/python/lib/python3.8/site-packages/paddle/fluid/framework.py:2138: DeprecationWarning: an integer is required (got type paddle.fluid.core_avx.op_proto_and_checker_maker.OpRole). Implicit conversion to integers using __int__ is deprecated, and may be removed in a future version of Python. + self.desc._set_attr(name, val) +/usr/local/python/lib/python3.8/site-packages/paddle/fluid/framework.py:541: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here. +Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations + elif dtype == np.bool: +. +---------------------------------------------------------------------- +Ran 1 test in 0.273s + +OK diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error2.log b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error2.log new file mode 100644 index 0000000..d9c024e --- /dev/null +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/logs/error2.log @@ -0,0 +1,12 @@ +/usr/local/python/lib/python3.8/site-packages/requests/__init__.py:102: RequestsDependencyWarning: urllib3 (1.26.6) or chardet (5.0.0)/charset_normalizer (2.0.1) doesn't match a supported version! + warnings.warn("urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " +/usr/local/python/lib/python3.8/site-packages/paddle/fluid/framework.py:2138: DeprecationWarning: an integer is required (got type paddle.fluid.core_avx.op_proto_and_checker_maker.OpRole). Implicit conversion to integers using __int__ is deprecated, and may be removed in a future version of Python. + self.desc._set_attr(name, val) +/usr/local/python/lib/python3.8/site-packages/paddle/fluid/framework.py:541: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here. +Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations + elif dtype == np.bool: +. +---------------------------------------------------------------------- +Ran 1 test in 0.243s + +OK -- Gitee From 23c537866e01bab41b8562184f4e1a53434d6276 Mon Sep 17 00:00:00 2001 From: "zongyi.chen" Date: Wed, 30 Nov 2022 16:10:28 +0800 Subject: [PATCH 3/6] Added a demo for calculating the median of two parties. --- ...44\270\255\344\275\215\346\225\260Demo.md" | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 "python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" diff --git "a/python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" "b/python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" new file mode 100644 index 0000000..0e36f9e --- /dev/null +++ "b/python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" @@ -0,0 +1,29 @@ +# 两方中位数Demo + +## 应用场景 + +两个数据方想要在不暴露自己的原始数据的前提下联合计算中位数。 + +## 算法原理及整体流程 + +1. 双方本地对自己的原始数据升序排列。 +2. 将双方排序好的数据按照aby3秘密共享的形式拆分，每方数据分为三份。 +3. 将三份数据交给三个不同的计算节点（本demo中的run.sh文件提供了单机模拟三方的示例，实际使用中拆开运行即可）。 +4. 双方使用Paddlefl提供的基础运算算子，按照[leetcode上的寻找两个有序数组的中位数算法（题解方法一：二分查找）](https://leetcode.cn/problems/median-of-two-sorted-arrays/solution/xun-zhao-liang-ge-you-xu-shu-zu-de-zhong-wei-s-114/)进行两方联合计算。 +5. 将秘密共享形式的计算结果交给数据合并方（由于本demo是单机模拟三方，这一步在实际使用中需要自行实现）。 +6. 重构计算结果。 + +## 代码执行过程 + +1. `python3 data.py` 生成双方原始数据，得到Input-P1.list和Input-P2.list。 +2. `python3 mid2_share.py` 将双方原始数据排序并拆分，得到data_C0_P1.npy、data_C1_P1.npy、data_C2_P1.npy、data_C0_P2.npy、data_C1_P2.npy、data_C2_P2.npy，如果双方数据量之和为偶数，还将得到data_C0_tmp.npy、data_C1_tmp.npy、data_C2_tmp.npy，同时会在命令行打印应当得到的中位数计算结果。 +3. 在真正的多方运算时需要分发第2步中的秘密数据（本demo不需要这一步）。 +4. `./run.sh` 运行单机模拟三方的脚本执行多方联合计算，logs文件夹中会生成运行日志，同时还会生成结果文件result_C0.npy、result_C1.npy、result_C2.npy。 +5. 在真正的多方运算时需要将第4步秘密共享形式的计算结果交给数据合并方（本demo不需要这一步）。 +6. `python3 reconstruct.py` 重构计算结果，并在命令行打印实际计算得到的中位数结果。 + +## 其他说明 + +1. 文件名中C代表计算方的编号，P代表数据方的编号，例如data_C0_P1.npy代表这份秘密来自数据方1，需要交给计算方0。 +2. 数据方编号由1开始，这是为了尽量保证寻找两个有序数组的中位数算法的核心代码与leetcode提供的算法一致，以增加代码可读性；计算方编号由0开始，这是为了尽量保证除了中位数算法核心代码以外的部分和paddlefl提供的/python/paddle_fl/mpc/tests/unittests中的其他示例对于参与方的描述一致。同时，这两部分代码的变量命名规范不同，中位数算法的核心代码使用驼峰命名法，其余部分和paddlefl一致使用下划线命名法。 +3. 由于paddlefl未提供除法算子，这里我们采用乘以倒数近似，即当双方数据总量为偶数n时，求第n/2和n/2+1个数据的平均值，需要除以2，data_Ci_tmp.npy文件中即存储秘密共享形式的0.5。 \ No newline at end of file -- Gitee From 7e519091f50f4fa1afa42a2d1dcade6048575224 Mon Sep 17 00:00:00 2001 From: "zongyi.chen" Date: Wed, 30 Nov 2022 16:11:11 +0800 Subject: [PATCH 4/6] Added a demo for calculating the median of two parties. --- .../paddle_fl/mpc/tests/demo/mid/mid2/readme.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename "python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" => python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md (100%) diff --git "a/python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" b/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md similarity index 100% rename from "python/paddle_fl/mpc/tests/demo/mid/mid2/\344\270\244\346\226\271\344\270\255\344\275\215\346\225\260Demo.md" rename to python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md -- Gitee From 7d16dc378dfe3f0f5b76530670388d8502ec2be2 Mon Sep 17 00:00:00 2001 From: "zongyi.chen" Date: Wed, 30 Nov 2022 16:21:25 +0800 Subject: [PATCH 5/6] Added a demo for calculating the median of two parties. --- python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md b/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md index 0e36f9e..9d51819 100644 --- a/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md @@ -18,12 +18,14 @@ 1. `python3 data.py` 生成双方原始数据，得到Input-P1.list和Input-P2.list。 2. `python3 mid2_share.py` 将双方原始数据排序并拆分，得到data_C0_P1.npy、data_C1_P1.npy、data_C2_P1.npy、data_C0_P2.npy、data_C1_P2.npy、data_C2_P2.npy，如果双方数据量之和为偶数，还将得到data_C0_tmp.npy、data_C1_tmp.npy、data_C2_tmp.npy，同时会在命令行打印应当得到的中位数计算结果。 3. 在真正的多方运算时需要分发第2步中的秘密数据（本demo不需要这一步）。 -4. `./run.sh` 运行单机模拟三方的脚本执行多方联合计算，logs文件夹中会生成运行日志，同时还会生成结果文件result_C0.npy、result_C1.npy、result_C2.npy。 -5. 在真正的多方运算时需要将第4步秘密共享形式的计算结果交给数据合并方（本demo不需要这一步）。 -6. `python3 reconstruct.py` 重构计算结果，并在命令行打印实际计算得到的中位数结果。 +4. `python3 flush.py` 清除redis中上一次残留的注册信息，以免下面运行的三方进程连接到错误的地方。 +5. `./run.sh` 运行单机模拟三方的脚本执行多方联合计算，logs文件夹中会生成运行日志，同时还会生成结果文件result_C0.npy、result_C1.npy、result_C2.npy。 +6. 在真正的多方运算时需要将第5步秘密共享形式的计算结果交给数据合并方（本demo不需要这一步）。 +7. `python3 reconstruct.py` 重构计算结果，并在命令行打印实际计算得到的中位数结果。 ## 其他说明 1. 文件名中C代表计算方的编号，P代表数据方的编号，例如data_C0_P1.npy代表这份秘密来自数据方1，需要交给计算方0。 2. 数据方编号由1开始，这是为了尽量保证寻找两个有序数组的中位数算法的核心代码与leetcode提供的算法一致，以增加代码可读性；计算方编号由0开始，这是为了尽量保证除了中位数算法核心代码以外的部分和paddlefl提供的/python/paddle_fl/mpc/tests/unittests中的其他示例对于参与方的描述一致。同时，这两部分代码的变量命名规范不同，中位数算法的核心代码使用驼峰命名法，其余部分和paddlefl一致使用下划线命名法。 -3. 由于paddlefl未提供除法算子，这里我们采用乘以倒数近似，即当双方数据总量为偶数n时，求第n/2和n/2+1个数据的平均值，需要除以2，data_Ci_tmp.npy文件中即存储秘密共享形式的0.5。 \ No newline at end of file +3. 由于paddlefl未提供除法算子，这里我们采用乘以倒数近似，即当双方数据总量为偶数n时，求第n/2和n/2+1个数据的平均值，需要除以2，data_Ci_tmp.npy文件中即存储秘密共享形式的0.5。 +4. 和paddlefl提供的test_op_base.py相比，注释掉了第70行代码，这一行代码会引起redis注册混乱，从而影响到三方模型运算；同时将multi_party_run改造成了三个不同的函数，以便三方分别调用单独生成进程。 \ No newline at end of file -- Gitee From ba087839b2615b4ebb112bc4b399308d32f93864 Mon Sep 17 00:00:00 2001 From: "zongyi.chen" Date: Wed, 30 Nov 2022 16:22:52 +0800 Subject: [PATCH 6/6] Added a demo for calculating the median of two parties. --- python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md b/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md index 9d51819..5433970 100644 --- a/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md +++ b/python/paddle_fl/mpc/tests/demo/mid/mid2/readme.md @@ -28,4 +28,4 @@ 1. 文件名中C代表计算方的编号，P代表数据方的编号，例如data_C0_P1.npy代表这份秘密来自数据方1，需要交给计算方0。 2. 数据方编号由1开始，这是为了尽量保证寻找两个有序数组的中位数算法的核心代码与leetcode提供的算法一致，以增加代码可读性；计算方编号由0开始，这是为了尽量保证除了中位数算法核心代码以外的部分和paddlefl提供的/python/paddle_fl/mpc/tests/unittests中的其他示例对于参与方的描述一致。同时，这两部分代码的变量命名规范不同，中位数算法的核心代码使用驼峰命名法，其余部分和paddlefl一致使用下划线命名法。 3. 由于paddlefl未提供除法算子，这里我们采用乘以倒数近似，即当双方数据总量为偶数n时，求第n/2和n/2+1个数据的平均值，需要除以2，data_Ci_tmp.npy文件中即存储秘密共享形式的0.5。 -4. 和paddlefl提供的test_op_base.py相比，注释掉了第70行代码，这一行代码会引起redis注册混乱，从而影响到三方模型运算；同时将multi_party_run改造成了三个不同的函数，以便三方分别调用单独生成进程。 \ No newline at end of file +4. 和paddlefl提供的原始test_op_base.py相比，本demo中注释掉了第70行代码，这一行代码会引起redis注册混乱，从而影响到三方模拟运算；同时将multi_party_run改造成了三个不同的函数，以便三方分别调用单独生成进程。 \ No newline at end of file -- Gitee