diff --git a/data.config b/data.config index f4076fc25867599d689e123f9b2301dfafd5779c..87e5346f10157328fc5ba44058e20598bcb148ab 100644 --- a/data.config +++ b/data.config @@ -1,27 +1,37 @@ [SERVER] 11.11.11.11 +[DOWNLOAD] +QE/6.4.1 https://github.com/QEF/q-e/archive/refs/tags/qe-6.4.1.tar.gz + [DEPENDENCY] -./jarvis -install gcc/9.3.1 com +./jarvis -install kgcc/9.3.1 com +module purge module use ./software/modulefiles -module load gcc9/9.3.1 +module load kgcc9/9.3.1 +export CC=`which gcc` +export CXX=`which g++` +export FC=`which gfortran` ./jarvis -install openmpi/4.1.2 gcc module load openmpi4/4.1.2 -./jarvis -install fftw/3.3.8 gcc +#test if mpi is normal +./jarvis -bench mpi +tar -xzvf ${JARVIS_DOWNLOAD}/qe-6.4.1.tar.gz [ENV] -# add gcc/mpi -export BLAS_LIBS="-L/usr/local/kml/lib/kblas/omp -lkblas" -export LAPACK_LIBS="-L/usr/local/kml/lib/ -lklapack_full" +module purge +module use ./software/modulefiles +module load kgcc9/9.3.1 +module load openmpi4/4.1.2 [APP] app_name = QE -build_dir = /home/HPCRunner/q-e-qe-6.5/ -binary_dir = /home/HPCRunner/QE-FFT/bin/ -case_dir = /home/HPCRunner/qe_large/ +build_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/ +binary_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/bin/ +case_dir = ${JARVIS_ROOT}/workloads/QE/qe-test [BUILD] -./configure --prefix=/home/HPCRunner/QE-FFT F90=gfortran F77=gfortran MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp +./configure F90=gfortran F77=gfortran MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp make -j 96 pwall make install @@ -29,11 +39,6 @@ make install make clean [RUN] -run = hpctool -o ./output -l detail -binary = pw.x -input scf.in -nodes = 1 - -[PERF] -perf = top -g -nsys = -y 5s -d 150s -ncu = --target-processes all --launch-skip 71434 --launch-count 1 \ No newline at end of file +run = mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -mca coll ^hcoll -mca btl ^vader,tcp,openib,uct -np 128 +binary = pw.x -input test_3.in +nodes = 1 \ No newline at end of file diff --git a/package/kgcc/10.3.1/install.sh b/package/kgcc/10.3.1/install.sh index 79fe13a6fed626317b220dfabdbfdec96336d7c3..62c38a94def52f4ceb6c53a42f9de05a26340478 100644 --- a/package/kgcc/10.3.1/install.sh +++ b/package/kgcc/10.3.1/install.sh @@ -1,5 +1,6 @@ #!/bin/bash set -x set -e +. ${DOWNLOAD_TOOL} -u https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2021.09-aarch64-linux.tar.gz cd ${JARVIS_TMP} -tar -xzvf ${JARVIS_DOWNLOAD}/gcc-10.3.1-2021.09-aarch64-linux.tar.gz -C $1 --strip-components=1 \ No newline at end of file +tar -xzvf ${JARVIS_DOWNLOAD}/gcc-10.3.1-2021.09-aarch64-linux.tar.gz -C $1 --strip-components=1 diff --git a/package/kgcc/9.3.1/install.sh b/package/kgcc/9.3.1/install.sh index 300c534ea7bcf1fc720a4d38c18ea662bb9e4663..10fd5e0434fac6cca58396dc7ec0b37c2243726f 100644 --- a/package/kgcc/9.3.1/install.sh +++ b/package/kgcc/9.3.1/install.sh @@ -1,5 +1,6 @@ #!/bin/bash set -x set -e +. ${DOWNLOAD_TOOL} -u https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-9.3.1-2021.03-aarch64-linux.tar.gz cd ${JARVIS_TMP} tar -xzvf ${JARVIS_DOWNLOAD}/gcc-9.3.1-2021.03-aarch64-linux.tar.gz -C $1 --strip-components=1 \ No newline at end of file diff --git a/package/openmpi/4.1.2/install.sh b/package/openmpi/4.1.2/install.sh index 581eae2a5db545803b5598e7e57bd3783a53e600..edd9624e471e08e759d9354e430a11d2542155f5 100644 --- a/package/openmpi/4.1.2/install.sh +++ b/package/openmpi/4.1.2/install.sh @@ -1,6 +1,7 @@ #!/bin/bash set -x set -e +. ${DOWNLOAD_TOOL} -u https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.2.tar.gz cd ${JARVIS_TMP} tar -xvf ${JARVIS_DOWNLOAD}/openmpi-4.1.2.tar.gz cd openmpi-4.1.2 diff --git a/src/downloadService.py b/src/downloadService.py index 4a644a76edc097cd766e44553d8915eb33d41fc6..ccf957f62ba8640d16bbfc856fd27e20ecb1c498 100644 --- a/src/downloadService.py +++ b/src/downloadService.py @@ -48,18 +48,12 @@ yum makecache url_list = url_info.split(' ') if len(url_list) < 2: continue - software_info = url_list[0].strip() url_link = url_list[1].strip() filename = os.path.basename(url_link) if len(url_list) == 3: filename = url_list[2].strip() filename_url_map[filename] = url_link - # create software directory - software_path = os.path.join(self.package_path, software_info) - self.tool.mkdirs(software_path) - # create install script - install_script = os.path.join(software_path, "install.sh") - self.tool.mkfile(install_script) + print(filename_url_map) # start download for filename, url in filename_url_map.items(): diff --git a/tunning/lammps/5Jun2019/License.txt b/tunning/lammps/5Jun2019/License.txt new file mode 100644 index 0000000000000000000000000000000000000000..f301e3f9c88b7c283e40d79aa54b2ac9ab3f586d --- /dev/null +++ b/tunning/lammps/5Jun2019/License.txt @@ -0,0 +1,26 @@ +Copyright (C) 2021 Huawei Technologies Co., Ltd. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/tunning/lammps/5Jun2019/Readme.md b/tunning/lammps/5Jun2019/Readme.md new file mode 100644 index 0000000000000000000000000000000000000000..f86449282b0b0c1fabe24aa6398e30ed64c3f619 --- /dev/null +++ b/tunning/lammps/5Jun2019/Readme.md @@ -0,0 +1,109 @@ +# LAMMPS + +## Description + +Lammps is a classical molecular dynamics software, which is often used to simulate materials. See [LAMMPS Home](https://lammps.sandia.gov/) for details. + +The current code repository is a series of patches and scripts for performance optimization of Lammps on Kunpeng chips. + +## Software Stack Used + +Compiler: Bisheng Compiler, see [Bisheng Home](https://support.huaweicloud.com/intl/en-us/ug-bisheng-kunpengdevps/kunpengbisheng_06_0001.html). + +MPI: Hyper MPI, see [Hyper MPI Home](https://support.huaweicloud.com/intl/en-us/usermanual-kunpenghpcs/userg_huaweimpi_0003.html). + +FFT: KML FFT, see [KML FFT Home](https://support.huaweicloud.com/intl/en-us/devg-kml-kunpengaccel/kunpengaccel_kml_16_0010.html). + +Lammps: lammps-5Jun2019.tar.gz + +- + +## How to Build + +### Installation Dependencies + +Install BiSheng compiler. + +Install Hyper MPI. + +Install KML FFT. + +### Unzip Lammps & Config USER-OMP + +``` +tar xvf lammps-5Jun2019.tar.gz + +cd lammps-5Jun19/ + +cp src/USER-OMP/hack_openmp_for_pgi_gcc9.sh ./src/ + +cp src/USER-OMP/hack_openmp_for_pgi_gcc9.sh ./src/SNAP/ + +cd src + +./hack_openmp_for_pgi_gcc9.sh + +cd USER-OMP/ + +./hack_openmp_for_pgi_gcc9.sh + +cd ../SNAP/ + +./hack_openmp_for_pgi_gcc9.sh + +cd ../ +``` + + + +### Install Patches + +``` +cp lammps-5Jun2019-aarch64-kunpeng-001.patch lammps-5Jun19/../ + +patch -p1 < lammps-5Jun2019-aarch64-kunpeng-001.patch +``` + + + +### Modify the KML FFT Path + +``` +vim lammps-5Jun19/src/MAKE/OPTIONS/Makefile.omp +``` + +FFT_INC = -DFFT_KML -I***/PATH/to/kml***/include +FFT_PATH = -L***/PATH/to/kml***/lib +FFT_LIB = -lkfft -lkm + +Change “***/PATH/to/kml***” to the real path of KML FFT. + + + +### Build Lammps + +``` +make no-all + +make clean-all + +make yes-std + +make yes-USER-OMP + +make no-lib + +make -j64 omp +``` + + + +## How to Run + +``` +mpirun --allow-run-as-root -mca pml ucx -mca btl ^vader,tcp,openib,uct -x UCX_TLS=self,sm --bind-to core --map-by socket --rank-by core -x UCX_BUILTIN_BCAST_ALGORITHM=3 -x UCX_BUILTIN_BARRIER_ALGORITHM=5 -x UCX_BUILTIN_ALLREDUCE_ALGORITHM=10 -np 128 /PATH/TO/lammps-5Jun19/src/lmp_omp -in equ.in +``` + +/PATH/TO/lammps-5Jun19/src/lmp_omp: path to the lmp_omp file. + +equ.in: input file to be calculated. \ No newline at end of file diff --git a/tunning/lammps/5Jun2019/lammps-5Jun2019-aarch64-kunpeng-001.patch b/tunning/lammps/5Jun2019/lammps-5Jun2019-aarch64-kunpeng-001.patch new file mode 100644 index 0000000000000000000000000000000000000000..fc9b4c9352581fa5d31ad42a824036cbd74c114b --- /dev/null +++ b/tunning/lammps/5Jun2019/lammps-5Jun2019-aarch64-kunpeng-001.patch @@ -0,0 +1,322 @@ +diff -x lmpgitversion.h -uNr origin/lammps-5Jun19/src/KSPACE/fft3d.cpp lammps-5Jun19/src/KSPACE/fft3d.cpp +--- origin/lammps-5Jun19/src/KSPACE/fft3d.cpp 2019-06-05 22:25:00.000000000 +0800 ++++ lammps-5Jun19/src/KSPACE/fft3d.cpp 2021-07-31 16:57:25.360000000 +0800 +@@ -31,6 +31,12 @@ + #include "kissfft.h" + #endif + ++#if defined(FFT_KML) ++ #define FFTW_FORWARD KML_FFT_FORWARD ++ #define FFTW_ESTIMATE KML_FFT_ESTIMATE ++ #define FFTW_BACKWARD KML_FFT_BACKWARD ++#endif ++ + #define MIN(A,B) ((A) < (B) ? (A) : (B)) + #define MAX(A,B) ((A) > (B) ? (A) : (B)) + +@@ -68,14 +74,14 @@ + { + int i,total,length,offset,num; + FFT_SCALAR norm; +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + FFT_SCALAR *out_ptr; + #endif + FFT_DATA *data,*copy; + + // system specific constants + +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + FFTW_API(plan) theplan; + #else + // nothing to do for other FFTs +@@ -111,7 +117,7 @@ + else + fftw(plan->plan_fast_backward,total/length,data,1,length,NULL,0,0); + */ +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + if (flag == -1) + theplan=plan->plan_fast_forward; + else +@@ -152,7 +158,7 @@ + else + fftw(plan->plan_mid_backward,total/length,data,1,length,NULL,0,0); + */ +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + if (flag == -1) + theplan=plan->plan_mid_forward; + else +@@ -193,7 +199,7 @@ + else + fftw(plan->plan_slow_backward,total/length,data,1,length,NULL,0,0); + */ +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + if (flag == -1) + theplan=plan->plan_slow_forward; + else +@@ -219,11 +225,11 @@ + if (flag == 1 && plan->scaled) { + norm = plan->norm; + num = plan->normnum; +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + out_ptr = (FFT_SCALAR *)out; + #endif + for (i = 0; i < num; i++) { +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + *(out_ptr++) *= norm; + *(out_ptr++) *= norm; + #elif defined(FFT_MKL) +@@ -570,7 +576,7 @@ + } + */ + +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + plan->plan_fast_forward = + FFTW_API(plan_many_dft)(1, &nfast,plan->total1/plan->length1, + NULL,&nfast,1,plan->length1, +@@ -682,7 +688,7 @@ + fftw_destroy_plan(plan->plan_fast_forward); + fftw_destroy_plan(plan->plan_fast_backward); + */ +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + FFTW_API(destroy_plan)(plan->plan_slow_forward); + FFTW_API(destroy_plan)(plan->plan_slow_backward); + FFTW_API(destroy_plan)(plan->plan_mid_forward); +@@ -780,7 +786,7 @@ + { + int i,num; + FFT_SCALAR norm; +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + FFT_SCALAR *data_ptr; + #endif + +@@ -798,7 +804,7 @@ + + // fftw3 and Dfti in MKL encode the number of transforms + // into the plan, so we cannot operate on a smaller data set. +-#if defined(FFT_MKL) || defined(FFT_FFTW3) ++#if defined(FFT_MKL) || defined(FFT_FFTW3) || defined(FFT_KML) + if ((total1 > nsize) || (total2 > nsize) || (total3 > nsize)) + return; + #endif +@@ -831,7 +837,7 @@ + fftw(plan->plan_slow_backward,total3/length3,data,1,0,NULL,0,0); + } + */ +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + FFTW_API(plan) theplan; + if (flag == -1) + theplan=plan->plan_fast_forward; +@@ -872,11 +878,11 @@ + if (flag == 1 && plan->scaled) { + norm = plan->norm; + num = MIN(plan->normnum,nsize); +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + data_ptr = (FFT_SCALAR *)data; + #endif + for (i = 0; i < num; i++) { +-#if defined(FFT_FFTW3) ++#if defined(FFT_FFTW3) || defined(FFT_KML) + *(data_ptr++) *= norm; + *(data_ptr++) *= norm; + #elif defined(FFT_MKL) +diff -x lmpgitversion.h -uNr origin/lammps-5Jun19/src/KSPACE/fft3d.h lammps-5Jun19/src/KSPACE/fft3d.h +--- origin/lammps-5Jun19/src/KSPACE/fft3d.h 2019-06-05 22:25:00.000000000 +0800 ++++ lammps-5Jun19/src/KSPACE/fft3d.h 2021-07-31 16:57:25.360000000 +0800 +@@ -53,6 +53,10 @@ + #include "fftw3.h" + typedef fftwf_complex FFT_DATA; + #define FFTW_API(function) fftwf_ ## function ++#elif defined(FFT_KML) ++#include "kfft.h" ++typedef kml_fft_complex FFT_DATA; ++#define FFTW_API(function) kml_fft_ ## function + + #else + +@@ -95,6 +99,11 @@ + typedef fftw_complex FFT_DATA; + #define FFTW_API(function) fftw_ ## function + ++#elif defined(FFT_KML) ++#include "kfft.h" ++typedef kml_fft_complex FFT_DATA; ++#define FFTW_API(function) kml_fft_ ## function ++ + #else + + /* use a stripped down version of kiss fft as default fft */ +@@ -146,7 +155,7 @@ + // fftw_plan plan_mid_backward; + //fftw_plan plan_slow_forward; + //fftw_plan plan_slow_backward; +-#elif defined(FFT_FFTW3) ++#elif defined(FFT_FFTW3) || defined(FFT_KML) + FFTW_API(plan) plan_fast_forward; + FFTW_API(plan) plan_fast_backward; + FFTW_API(plan) plan_mid_forward; +diff -x lmpgitversion.h -uNr origin/lammps-5Jun19/src/MAKE/OPTIONS/Makefile.omp lammps-5Jun19/src/MAKE/OPTIONS/Makefile.omp +--- origin/lammps-5Jun19/src/MAKE/OPTIONS/Makefile.omp 2019-06-05 22:25:00.000000000 +0800 ++++ lammps-5Jun19/src/MAKE/OPTIONS/Makefile.omp 2021-07-31 17:14:20.020000000 +0800 +@@ -6,14 +6,14 @@ + # compiler/linker settings + # specify flags and libraries needed for your compiler + +-CC = mpicxx +-CCFLAGS = -g -O3 -restrict -fopenmp ++CC = mpicxx -stdlib=libc++ ++CCFLAGS = -O3 -fopenmp -march=armv8.2-a -mcpu=tsv110 -fuse-ld=lld -flto=full + SHFLAGS = -fPIC + DEPFLAGS = -M + +-LINK = mpicxx +-LINKFLAGS = -g -O -fopenmp +-LIB = ++LINK = mpicxx -stdlib=libc++ ++LINKFLAGS = -O3 -fopenmp -march=armv8.2-a -mcpu=tsv110 -fuse-ld=lld -flto=full ++LIB = -lmathlib -lm + SIZE = size + + ARCHIVE = ar +@@ -50,9 +50,9 @@ + # PATH = path for FFT library + # LIB = name of FFT library + +-FFT_INC = +-FFT_PATH = +-FFT_LIB = ++FFT_INC = -DFFT_KML -I/PATH/to/kml/include ++FFT_PATH = -L/PATH/to/kml/lib ++FFT_LIB = -lkfft -lkm + + # JPEG and/or PNG library + # see discussion in Section 2.2 (step 7) of manual +diff -x lmpgitversion.h -uNr origin/lammps-5Jun19/src/USER-OMP/npair_full_bin_ghost_omp.cpp lammps-5Jun19/src/USER-OMP/npair_full_bin_ghost_omp.cpp +--- origin/lammps-5Jun19/src/USER-OMP/npair_full_bin_ghost_omp.cpp 2021-07-31 17:18:43.750000000 +0800 ++++ lammps-5Jun19/src/USER-OMP/npair_full_bin_ghost_omp.cpp 2021-07-31 16:57:25.370000000 +0800 +@@ -44,7 +44,7 @@ + + NPAIR_OMP_INIT; + #if defined(_OPENMP) +-#pragma omp parallel default(shared) ++//#pragma omp parallel default(shared) + #endif + NPAIR_OMP_SETUP(nall); + +diff -x lmpgitversion.h -uNr origin/lammps-5Jun19/src/USER-OMP/pair_lj_cut_tip4p_long_omp.cpp lammps-5Jun19/src/USER-OMP/pair_lj_cut_tip4p_long_omp.cpp +--- origin/lammps-5Jun19/src/USER-OMP/pair_lj_cut_tip4p_long_omp.cpp 2021-07-31 17:18:44.030000000 +0800 ++++ lammps-5Jun19/src/USER-OMP/pair_lj_cut_tip4p_long_omp.cpp 2021-07-31 16:57:25.370000000 +0800 +@@ -93,7 +93,7 @@ + const int inum = list->inum; + + #if defined(_OPENMP) +-#pragma omp parallel default(shared) ++//#pragma omp parallel default(shared) + #endif + { + int ifrom, ito, tid; +@@ -213,7 +213,7 @@ + + jlist = firstneigh[i]; + jnum = numneigh[i]; +- fxtmp=fytmp=fztmp=0.0; ++ //fxtmp=fytmp=fztmp=0.0; + + for (jj = 0; jj < jnum; jj++) { + j = jlist[jj]; +@@ -227,6 +227,13 @@ + rsq = delx*delx + dely*dely + delz*delz; + jtype = type[j]; + ++ #define const_offsetC 16 ++ if (jj < jnum - const_offsetC) { ++ __builtin_prefetch(&x[jlist[jj+const_offsetC]&NEIGHMASK].x); ++ __builtin_prefetch(&x[jlist[jj+const_offsetC]&NEIGHMASK].z); ++ } ++ __builtin_prefetch(&jlist[jj+2*const_offsetC]); ++ + // LJ interaction based on true rsq + + if (rsq < cut_ljsq[itype][jtype]) { +@@ -235,9 +242,13 @@ + forcelj = r6inv * (lj1[itype][jtype]*r6inv - lj2[itype][jtype]); + forcelj *= factor_lj * r2inv; + +- fxtmp += delx*forcelj; +- fytmp += dely*forcelj; +- fztmp += delz*forcelj; ++ //fxtmp += delx*forcelj; ++ //fytmp += dely*forcelj; ++ //fztmp += delz*forcelj; ++ f[i].x += delx*forcelj; ++ f[i].y += dely*forcelj; ++ f[i].z += delz*forcelj; ++ + f[j].x -= delx*forcelj; + f[j].y -= dely*forcelj; + f[j].z -= delz*forcelj; +@@ -336,16 +347,19 @@ + // preserves total force and torque on water molecule + // virial = sum(r x F) where each water's atoms are near xi and xj + // vlist stores 2,4,6 atoms whose forces contribute to virial +- ++ __builtin_prefetch(&f[j].x, 1, 3); + if (VFLAG) { + n = 0; + key = 0; + } + + if (itype != typeO) { +- fxtmp += delx * cforce; +- fytmp += dely * cforce; +- fztmp += delz * cforce; ++ //fxtmp += delx * cforce; ++ //fytmp += dely * cforce; ++ //fztmp += delz * cforce; ++ f[i].x += delx * cforce; ++ f[i].y += dely * cforce; ++ f[i].z += delz * cforce; + + if (VFLAG) { + v[0] = x[i].x * delx * cforce; +@@ -372,9 +386,13 @@ + fHy = 0.5*alpha * fdy; + fHz = 0.5*alpha * fdz; + +- fxtmp += fOx; +- fytmp += fOy; +- fztmp += fOz; ++ //fxtmp += fOx; ++ //fytmp += fOy; ++ //fztmp += fOz; ++ f[i].x += fOx; ++ f[i].y += fOy; ++ f[i].z += fOz; ++ + + f[iH1].x += fHx; + f[iH1].y += fHy; +@@ -470,9 +488,9 @@ + } + } + } +- f[i].x += fxtmp; +- f[i].y += fytmp; +- f[i].z += fztmp; ++ //f[i].x += fxtmp; ++ //f[i].y += fytmp; ++ //f[i].z += fztmp; + } + } +