Skip to content

Commit 1c4dc78

Browse files
author
Avirup Sircar
committed
Modify in KE Hamiltonian energy eval
1 parent ae37fb6 commit 1c4dc78

File tree

10 files changed

+321
-157
lines changed

10 files changed

+321
-157
lines changed

‎analysis/classicalEnrichmentComparison/PSPnonlocal/KSDFTOrthoEFEUniformQuad/TestKohnShamDft.cpp‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,21 @@ int main(int argc, char** argv)
513513
numElectrons += (size_type)(std::abs(i));
514514
}
515515

516+
if (numWantedEigenvalues <= numElectrons / 2.0 ||
517+
numWantedEigenvalues == 0)
518+
{
519+
rootCout << " Warning: User has requested the number of Kohn-Sham wavefunctions to be less than or"
520+
"equal to half the number of electrons in the system. Setting the Kohn-Sham wavefunctions"
521+
"to half the number of electrons with a 20 percent buffer to avoid convergence issues in"
522+
"SCF iterations" << std::endl;
523+
numWantedEigenvalues = (numElectrons / 2.0) + std::max((0.2) * (numElectrons / 2.0), 20.0);
524+
525+
// start with 17-20% buffer in GPUs to leave room for additional modifications
526+
// due to block size restrictions
527+
528+
rootCout << " Setting the number of Kohn-Sham wave functions to be " << numWantedEigenvalues << std::endl;
529+
}
530+
516531
std::vector<std::string> fieldNames{"orbital","vtotal","density"};
517532
std::vector<std::string> metadataNames{ "symbol", "Z", "charge", "NR" };
518533
std::shared_ptr<atoms::AtomSphericalDataContainer> atomSphericalDataContainer =
@@ -958,6 +973,8 @@ int main(int argc, char** argv)
958973
linAlgOpContext,
959974
true);
960975

976+
utils::printCurrentMemoryUsage(comm, "Hamiltonian Basis overlap");
977+
961978
// quadrature::QuadratureRuleAttributes quadAttrGaussEigen(quadrature::QuadratureFamily::GAUSS,true,feOrderEigen + 1);
962979

963980
// std::shared_ptr<basis::FEBasisDataStorage<double, Host>> cfeBasisDataStorageGaussEigen =
@@ -996,6 +1013,8 @@ int main(int argc, char** argv)
9961013
true);
9971014

9981015
p.registerEnd("Hamiltonian Basis overlap eval");
1016+
utils::printCurrentMemoryUsage(comm, "Hamiltonian Basis overlap , overlap for inv");
1017+
9991018
p.registerStart("Hamiltonian Basis overlap inverse eval");
10001019

10011020
std::shared_ptr<linearAlgebra::OperatorContext<double,
@@ -1012,8 +1031,9 @@ int main(int argc, char** argv)
10121031
*cfeBasisDataStorageGLLEigen,
10131032
linAlgOpContext);
10141033

1015-
p.registerEnd("Hamiltonian Basis overlap inverse eval");
1016-
utils::printCurrentMemoryUsage(comm, "Hamiltonian Basis overlap and inv");
1034+
p.registerEnd("Hamiltonian Basis overlap inverse eval");
1035+
utils::printCurrentMemoryUsage(comm, "Hamiltonian Basis overlap , overlap for inv and inv");
1036+
10171037
p.registerStart("Kohn Sham DFT Class Init");
10181038
ksdft::KohnShamDFT<double,
10191039
double,
@@ -1022,6 +1042,8 @@ int main(int argc, char** argv)
10221042
Host,
10231043
dim>* dftefeSolve = nullptr;
10241044

1045+
utils::printCurrentMemoryUsage(comm, "Before Kohn Sham DFT Class Init");
1046+
10251047
if(isNumericalNuclearSolve && !isDeltaRhoPoissonSolve)
10261048
{
10271049
utils::throwException(false, "Option not there for KohnShamDFT class creation.");
@@ -1130,7 +1152,7 @@ int main(int argc, char** argv)
11301152
utils::throwException(false, "Option not there for KohnShamDFT class creation.");
11311153
}
11321154
p.registerEnd("Kohn Sham DFT Class Init");
1133-
utils::printCurrentMemoryUsage(comm, "Kohn Sham DFT Class Init");
1155+
utils::printCurrentMemoryUsage(comm, "After Kohn Sham DFT Class Init");
11341156
p.print();
11351157

11361158
pTot.registerEnd("Initilization");

‎src/atoms/AtomSphericalDataEnrichmentJSON.cpp‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,8 @@ namespace dftefe
194194
{
195195
utils::throwException(false,
196196
metadataName +
197-
" metadataName type not found.");
197+
" metadataName type not found for " +
198+
fileName + ".");
198199
}
199200

200201
simdjson::dom::element elem = typeRes.value();
@@ -615,7 +616,7 @@ namespace dftefe
615616
cutoffId = j;
616617
break;
617618
}
618-
if (1 - intgl[j] > 1.e-3)
619+
if (1 - intgl[j] > 5.e-3)
619620
{
620621
cutoffId1 = j;
621622
break;
@@ -653,9 +654,16 @@ namespace dftefe
653654
}
654655
else if (d_PSPorAE == "AE")
655656
{
657+
double smoothness = 1.01;
658+
if (radialPoints[std::max(cutoffId, cutoffId1)] > 10.0)
659+
{
660+
cutOffInfoVec[i].first =
661+
10 * (smoothness / (1 + smoothness));
662+
}
656663
cutOffInfoVec[i].first =
657-
radialPoints[std::max(cutoffId, cutoffId1)] * (1. / 2);
658-
cutOffInfoVec[i].second = 1.01;
664+
radialPoints[std::max(cutoffId, cutoffId1)] *
665+
(smoothness / (1 + smoothness));
666+
cutOffInfoVec[i].second = smoothness;
659667
}
660668
else
661669
{

‎src/basis/EnrichmentClassicalInterfaceSpherical.t.cpp‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -549,9 +549,11 @@ namespace dftefe
549549
global_size_type maxEnrich = 0;
550550
global_size_type minEnrich = 0;
551551
global_size_type avgEnrich = 0;
552-
cell = d_triangulation->beginLocal();
553-
endc = d_triangulation->endLocal();
554-
cellIndex = 0;
552+
global_size_type maxTotalEnrichInProc =
553+
d_enrichmentIdsPartition->nLocalEnrichmentIds();
554+
cell = d_triangulation->beginLocal();
555+
endc = d_triangulation->endLocal();
556+
cellIndex = 0;
555557
for (; cell != endc; cell++)
556558
{
557559
global_size_type numEnrichInCell =
@@ -587,20 +589,29 @@ namespace dftefe
587589
&avgEnrich,
588590
1,
589591
utils::mpi::Types<global_size_type>::getMPIDatatype(),
590-
utils::mpi::MPISum,
592+
utils::mpi::MPIMax,
591593
comm);
592594

593-
avgEnrich /= numProcs;
595+
utils::mpi::MPIAllreduce<memorySpace>(
596+
utils::mpi::MPIInPlace,
597+
&maxTotalEnrichInProc,
598+
1,
599+
utils::mpi::Types<global_size_type>::getMPIDatatype(),
600+
utils::mpi::MPIMax,
601+
comm);
594602

595603
rootCout << "Maximum " << fieldName
596-
<< " Enrichment Ids In a Cell in Processor: " << maxEnrich
604+
<< " Enrichment Ids In a Cell in a Processor: " << maxEnrich
597605
<< "\n";
598606
rootCout << "Minimum " << fieldName
599-
<< " Enrichment Ids In a Cell in Processor: " << minEnrich
607+
<< " Enrichment Ids In a Cell in a Processor: " << minEnrich
600608
<< "\n";
601-
rootCout << "Average " << fieldName
602-
<< " Enrichment Ids In a Cell In Processor: " << avgEnrich
609+
rootCout << "Max Average " << fieldName
610+
<< " Enrichment Ids In a Cell in a Processor: " << avgEnrich
603611
<< "\n";
612+
rootCout << "Maximum " << fieldName
613+
<< " Total Enrichment Ids In a Processor: "
614+
<< maxTotalEnrichInProc << "\n";
604615

605616
rootCout
606617
<< "Completed creating Orthogonalized EnrichmentClassicalInterfaceSpherical for "
@@ -725,8 +736,10 @@ namespace dftefe
725736
global_size_type maxEnrich = 0;
726737
global_size_type minEnrich = 0;
727738
global_size_type avgEnrich = 0;
728-
size_type cellIndex = 0;
729-
cell = d_triangulation->beginLocal();
739+
global_size_type maxTotalEnrichInProc =
740+
d_enrichmentIdsPartition->nLocalEnrichmentIds();
741+
size_type cellIndex = 0;
742+
cell = d_triangulation->beginLocal();
730743
for (; cell != endc; cell++)
731744
{
732745
global_size_type numEnrichInCell =
@@ -765,15 +778,27 @@ namespace dftefe
765778
utils::mpi::MPISum,
766779
comm);
767780

781+
utils::mpi::MPIAllreduce<memorySpace>(
782+
utils::mpi::MPIInPlace,
783+
&maxTotalEnrichInProc,
784+
1,
785+
utils::mpi::Types<global_size_type>::getMPIDatatype(),
786+
utils::mpi::MPIMax,
787+
comm);
788+
768789
avgEnrich /= numProcs;
769790

770791
rootCout << "Maximum " << fieldName
771-
<< " Enrichment Ids In a Processor: " << maxEnrich << "\n";
792+
<< " Enrichment Ids In a Cell in Processor: " << maxEnrich
793+
<< "\n";
772794
rootCout << "Minimum " << fieldName
773795
<< " Enrichment Ids In a Cell in Processor: " << minEnrich
774796
<< "\n";
775797
rootCout << "Average " << fieldName
776-
<< " Enrichment Ids In a Processor: " << avgEnrich << "\n";
798+
<< " Enrichment Ids In a Cell Processor: " << avgEnrich << "\n";
799+
rootCout << "Maximum " << fieldName
800+
<< " Total Enrichment Ids In a Processor: "
801+
<< maxTotalEnrichInProc << "\n";
777802

778803
rootCout
779804
<< "Completed creating Pristine EnrichmentClassicalInterfaceSpherical for "

‎src/basis/OEFEAtomBlockOverlapInvOpContextGLL.t.cpp‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -725,10 +725,9 @@ namespace dftefe
725725
d_diagonalInv, 1);
726726

727727
// Now form the enrichment block matrix.
728-
std::shared_ptr<utils::MemoryStorage<ValueTypeOperator, memorySpace>>
729-
basisOverlapInvEnrichmentBlockExact = std::make_shared<
730-
utils::MemoryStorage<ValueTypeOperator, memorySpace>>(
731-
d_nglobalEnrichmentIds * d_nglobalEnrichmentIds);
728+
// utils::MemoryStorage<ValueTypeOperator, memorySpace>
729+
// basisOverlapInvEnrichmentBlockExact(
730+
// d_nglobalEnrichmentIds * d_nglobalEnrichmentIds);
732731

733732
utils::MemoryStorage<ValueTypeOperator, memorySpace>
734733
basisOverlapInvEnrichmentBlock(d_nglobalEnrichmentIds *
@@ -746,13 +745,13 @@ namespace dftefe
746745
{
747746
for (unsigned int k = 0; k < nCellEnrichmentDofs; k++)
748747
{
749-
*(basisOverlapInvEnrichmentBlockExact->data() +
750-
enrichmentVecInCell[j] * d_nglobalEnrichmentIds +
751-
enrichmentVecInCell[k]) +=
752-
*(NiNjInAllCells.data() + cumulativeBasisDataInCells +
753-
(numCellClassicalDofs + nCellEnrichmentDofs) *
754-
(numCellClassicalDofs + j) +
755-
numCellClassicalDofs + k);
748+
// *(basisOverlapInvEnrichmentBlockExact.data() +
749+
// enrichmentVecInCell[j] * d_nglobalEnrichmentIds +
750+
// enrichmentVecInCell[k]) +=
751+
// *(NiNjInAllCells.data() + cumulativeBasisDataInCells +
752+
// (numCellClassicalDofs + nCellEnrichmentDofs) *
753+
// (numCellClassicalDofs + j) +
754+
// numCellClassicalDofs + k);
756755

757756
basis::EnrichmentIdAttribute eIdAttrj =
758757
efeBDH->getEnrichmentIdsPartition()
@@ -779,31 +778,32 @@ namespace dftefe
779778
cellId += 1;
780779
}
781780

782-
int err = utils::mpi::MPIAllreduce<memorySpace>(
783-
utils::mpi::MPIInPlace,
784-
basisOverlapInvEnrichmentBlockExact->data(),
785-
basisOverlapInvEnrichmentBlockExact->size(),
786-
utils::mpi::MPIDouble,
787-
utils::mpi::MPISum,
788-
d_feBasisManager->getMPIPatternP2P()->mpiCommunicator());
789-
std::pair<bool, std::string> mpiIsSuccessAndMsg =
790-
utils::mpi::MPIErrIsSuccessAndMsg(err);
791-
utils::throwException(mpiIsSuccessAndMsg.first,
792-
"MPI Error:" + mpiIsSuccessAndMsg.second);
793-
794-
linearAlgebra::blasLapack::inverse<ValueTypeOperator, memorySpace>(
795-
d_nglobalEnrichmentIds,
796-
basisOverlapInvEnrichmentBlockExact->data(),
797-
*(d_diagonalInv.getLinAlgOpContext()));
781+
// int err = utils::mpi::MPIAllreduce<memorySpace>(
782+
// utils::mpi::MPIInPlace,
783+
// basisOverlapInvEnrichmentBlockExact.data(),
784+
// basisOverlapInvEnrichmentBlockExact.size(),
785+
// utils::mpi::MPIDouble,
786+
// utils::mpi::MPISum,
787+
// d_feBasisManager->getMPIPatternP2P()->mpiCommunicator());
788+
// std::pair<bool, std::string> mpiIsSuccessAndMsg =
789+
// utils::mpi::MPIErrIsSuccessAndMsg(err);
790+
// utils::throwException(mpiIsSuccessAndMsg.first,
791+
// "MPI Error:" + mpiIsSuccessAndMsg.second);
792+
793+
// linearAlgebra::blasLapack::inverse<ValueTypeOperator, memorySpace>(
794+
// d_nglobalEnrichmentIds,
795+
// basisOverlapInvEnrichmentBlockExact.data(),
796+
// *(d_diagonalInv.getLinAlgOpContext()));
798797

799-
err = utils::mpi::MPIAllreduce<memorySpace>(
798+
int err = utils::mpi::MPIAllreduce<memorySpace>(
800799
utils::mpi::MPIInPlace,
801800
basisOverlapInvEnrichmentBlock.data(),
802801
basisOverlapInvEnrichmentBlock.size(),
803802
utils::mpi::MPIDouble,
804803
utils::mpi::MPISum,
805804
d_feBasisManager->getMPIPatternP2P()->mpiCommunicator());
806-
mpiIsSuccessAndMsg = utils::mpi::MPIErrIsSuccessAndMsg(err);
805+
std::pair<bool, std::string> mpiIsSuccessAndMsg =
806+
utils::mpi::MPIErrIsSuccessAndMsg(err);
807807
utils::throwException(mpiIsSuccessAndMsg.first,
808808
"MPI Error:" + mpiIsSuccessAndMsg.second);
809809

@@ -863,11 +863,11 @@ namespace dftefe
863863
// for (int i = 0; i < basisOverlapInvEnrichmentBlock.size(); i++)
864864
// {
865865
// *(basisOverlapInvEnrichmentBlock.data() + i) =
866-
// *(basisOverlapInvEnrichmentBlockExact->data() + i) -
866+
// *(basisOverlapInvEnrichmentBlockExact.data() + i) -
867867
// *(basisOverlapInvEnrichmentBlock.data() + i);
868868

869-
// normMInvexact += *(basisOverlapInvEnrichmentBlockExact->data()
870-
// + i) * *(basisOverlapInvEnrichmentBlockExact->data() + i);
869+
// normMInvexact += *(basisOverlapInvEnrichmentBlockExact.data()
870+
// + i) * *(basisOverlapInvEnrichmentBlockExact.data() + i);
871871
// }
872872
// normMInvexact = std::sqrt(normMInvexact);
873873

0 commit comments

Comments
 (0)