From 1419f573f70bfb42b5afd967f397741d7051dd63 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 11:33:38 -0700 Subject: [PATCH 01/69] Added '/DET' flag to ILASM for deterministic compilations --- src/coreclr/dlls/mscorpe/pewriter.cpp | 5 ++ src/coreclr/ilasm/assem.cpp | 7 +++ src/coreclr/ilasm/assembler.h | 4 ++ src/coreclr/ilasm/main.cpp | 4 ++ src/coreclr/ilasm/writer.cpp | 86 +++++++++++++++++++++++++++ src/coreclr/inc/iceefilegen.h | 2 + 6 files changed, 108 insertions(+) diff --git a/src/coreclr/dlls/mscorpe/pewriter.cpp b/src/coreclr/dlls/mscorpe/pewriter.cpp index 2f40d3cd5b593..8b942bac008e5 100644 --- a/src/coreclr/dlls/mscorpe/pewriter.cpp +++ b/src/coreclr/dlls/mscorpe/pewriter.cpp @@ -816,6 +816,11 @@ HRESULT PEWriter::Init(PESectionMan *pFrom, DWORD createFlags, LPCWSTR seedFileN m_ntHeaders->FileHeader.Characteristics |= VAL16(IMAGE_FILE_RELOCS_STRIPPED); } + if (createFlags & ICEE_CREATE_FILE_DET) + { + m_ntHeaders->FileHeader.TimeDateStamp = VAL32(0); + } + // Linker version should be consistent with current VC level m_ntHeaders->OptionalHeader.MajorLinkerVersion = 11; m_ntHeaders->OptionalHeader.MinorLinkerVersion = 0; diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index c4b8c6aea0e45..aaac49098609c 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -28,6 +28,7 @@ Assembler::Assembler() { m_pDisp = NULL; m_pEmitter = NULL; + m_pInternalEmitForDeterministicMvid = NULL; m_pImporter = NULL; char* pszFQN = new char[16]; @@ -107,6 +108,7 @@ Assembler::Assembler() m_fGeneratePDB = FALSE; m_fIsMscorlib = FALSE; m_fOptimize = FALSE; + m_fDeterministic = FALSE; m_tkSysObject = 0; m_tkSysString = 0; m_tkSysValue = 0; @@ -207,6 +209,11 @@ Assembler::~Assembler() m_pEmitter->Release(); m_pEmitter = NULL; } + if (m_pInternalEmitForDeterministicMvid != NULL) + { + m_pInternalEmitForDeterministicMvid->Release(); + m_pInternalEmitForDeterministicMvid = NULL; + } if (m_pPortablePdbWriter != NULL) { delete m_pPortablePdbWriter; diff --git a/src/coreclr/ilasm/assembler.h b/src/coreclr/ilasm/assembler.h index b80ef9a16d3c7..38f1fd0d03295 100644 --- a/src/coreclr/ilasm/assembler.h +++ b/src/coreclr/ilasm/assembler.h @@ -752,6 +752,7 @@ class Assembler { BOOL m_fIsMscorlib; BOOL m_fTolerateDupMethods; BOOL m_fOptimize; + BOOL m_fDeterministic; mdToken m_tkSysObject; mdToken m_tkSysString; mdToken m_tkSysValue; @@ -760,6 +761,7 @@ class Assembler { IMetaDataDispenserEx2 *m_pDisp; IMetaDataEmit3 *m_pEmitter; + IMDInternalEmit *m_pInternalEmitForDeterministicMvid; ICeeFileGen *m_pCeeFileGen; IMetaDataImport2 *m_pImporter; // Import interface. HCEEFILE m_pCeeFile; @@ -1256,6 +1258,8 @@ class Assembler { }; +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize); + #endif // Assember_h #ifdef _MSC_VER diff --git a/src/coreclr/ilasm/main.cpp b/src/coreclr/ilasm/main.cpp index d50c7da252519..4b6279cb7e0a0 100644 --- a/src/coreclr/ilasm/main.cpp +++ b/src/coreclr/ilasm/main.cpp @@ -320,6 +320,10 @@ extern "C" int _cdecl wmain(int argc, _In_ WCHAR **argv) { pAsm->m_fOptimize = TRUE; } + else if (!_stricmp(szOpt, "DET")) + { + pAsm->m_fDeterministic = TRUE; + } else if (!_stricmp(szOpt, "X64")) { pAsm->m_dwCeeFileFlags &= ~ICEE_CREATE_MACHINE_MASK; diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 0203a99cd3b7f..4312dacf5be18 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -43,6 +43,34 @@ HRESULT Assembler::InitMetaData() if(FAILED(hr = m_pEmitter->QueryInterface(IID_IMetaDataImport2, (void**)&m_pImporter))) goto exit; + if (m_fDeterministic) + { + // + // In deterministic mode, the MVID will need to be stabilized for the metadata scope that + // was created above, and the ChangeMvid service that makes this possible is only available + // on the IMDInternalEmit interface. + // + // When the CLSID_CorMetaDataDispenser instance above has activated against a current + // clr.dll (which is the only supported configuration for the determinism feature), it is + // guaranteed that "m_pEmitter" is implemented by the RegMeta object that was created + // during the DefineScope call above, and it is guaranteed that this same RegMeta object + // also implements the required IMDInternalEmit interface. + // + // Any failure is unexpected and catastrophic, so print a noisy message and return an + // error (which generally fails the entire ilasm operation) if any failure occurs. + // + + hr = m_pEmitter->QueryInterface(IID_IMDInternalEmit, (void**)&m_pInternalEmitForDeterministicMvid); + + if (FAILED(hr) || (m_pInternalEmitForDeterministicMvid == NULL)) + { + fprintf(stderr, "Unexpected: Failed to query the required MVID determinism interface: %X\n",hr); + hr = E_FAIL; + goto exit; + } + } + + if (m_fGeneratePDB) { m_pPortablePdbWriter = new PortablePdbWriter(); @@ -1590,6 +1618,64 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) exit: return hr; } + +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +{ + NTSTATUS status; + + BCRYPT_ALG_HANDLE algHandle = NULL; + BCRYPT_HASH_HANDLE hashHandle = NULL; + + BYTE hash[32]; // 256 bits + DWORD hashLength = 0; + DWORD resultLength = 0; + status = BCryptOpenAlgorithmProvider(&algHandle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_HASH_REUSABLE_FLAG); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + status = BCryptGetProperty(algHandle, BCRYPT_HASH_LENGTH, (PBYTE)&hashLength, sizeof(hashLength), &resultLength, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + if (hashLength != 32) + { + status = STATUS_NO_MEMORY; + goto cleanup; + } + status = BCryptCreateHash(algHandle, &hashHandle, NULL, 0, NULL, 0, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + status = BCryptHashData(hashHandle, pSrc, srcSize, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + status = BCryptFinishHash(hashHandle, hash, hashLength, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + memcpy(pDst, hash, min(hashLength, dstSize)); + status = STATUS_SUCCESS; + +cleanup: + if (NULL != hashHandle) + { + BCryptDestroyHash(hashHandle); + } + if(NULL != algHandle) + { + BCryptCloseAlgorithmProvider(algHandle, 0); + } + return status; +} + #ifdef _PREFAST_ #pragma warning(pop) #endif diff --git a/src/coreclr/inc/iceefilegen.h b/src/coreclr/inc/iceefilegen.h index f0fb5b091c9ca..a32d248da4c17 100644 --- a/src/coreclr/inc/iceefilegen.h +++ b/src/coreclr/inc/iceefilegen.h @@ -67,6 +67,8 @@ typedef HRESULT (__stdcall * PFN_DestroyICeeFileGen)(ICeeFileGen ** ceeFileGen); #define ICEE_CREATE_MACHINE_ARM 0x00000800 // Create a IMAGE_FILE_MACHINE_ARMNT #define ICEE_CREATE_MACHINE_ARM64 0x00001000 // Create a IMAGE_FILE_MACHINE_ARM64 +#define ICEE_CREATE_FILE_DET 0x00100000 // Creates a deterministic PE + // Pass this to CreateCeeFileEx to create a pure IL Exe or DLL #define ICEE_CREATE_FILE_PURE_IL ICEE_CREATE_FILE_PE32 | \ ICEE_CREATE_FILE_CORMAIN_STUB | \ From d3b329cae186c2096fa72237815c61753f9c6d54 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 11:39:01 -0700 Subject: [PATCH 02/69] Using 'm_pInternalEmitForDeterministicMvid' --- src/coreclr/ilasm/writer.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 4312dacf5be18..4c06a0ee58bed 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1534,6 +1534,18 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr)) goto exit; } + if (m_fDeterministic) + { + // In deterministic mode, the MVID needs to be stabilized for the metadata scope that was + // created in Assembler::InitMetaData, and it is guaranteed that the IMDInternalEmit for + // that scope was already acquired immediately after that scope was created. + _ASSERTE(m_pInternalEmitForDeterministicMvid != NULL); + GUID mvid; + hr = Sha256Hash(metaData, metaDataSize, (BYTE*)&mvid, sizeof(GUID)); + if (FAILED(hr)) goto exit; + m_pInternalEmitForDeterministicMvid->ChangeMvid(mvid); + } + if(bClock) bClock->cFilegenBegin = GetTickCount(); // actually output the meta-data if (FAILED(hr=m_pCeeFileGen->EmitMetaDataAt(m_pCeeFile, m_pEmitter, m_pILSection, metaDataOffset, metaData, metaDataSize))) goto exit; From 0357bf1509f18e235c771f0e9cd7065c05b9eaa0 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 12:41:40 -0700 Subject: [PATCH 03/69] Removing use of BCRYPT APIs. Added TODOs. --- src/coreclr/ilasm/portable_pdb.cpp | 10 +++++ src/coreclr/ilasm/portable_pdb.h | 2 + src/coreclr/ilasm/writer.cpp | 70 +++++++----------------------- 3 files changed, 28 insertions(+), 54 deletions(-) diff --git a/src/coreclr/ilasm/portable_pdb.cpp b/src/coreclr/ilasm/portable_pdb.cpp index 5e56e55eb716e..8e52bd2ac8161 100644 --- a/src/coreclr/ilasm/portable_pdb.cpp +++ b/src/coreclr/ilasm/portable_pdb.cpp @@ -120,6 +120,16 @@ ULONG PortablePdbWriter::GetTimestamp() return m_pdbStream.id.pdbTimeStamp; } +void PortablePdbWriter::SetGuid(REFGUID newGuid) +{ + m_pdbStream.id.pdbGuid = newGuid; +} + +void PortablePdbWriter::SetTimestamp(const ULONG newTimestamp) +{ + m_pdbStream.id.pdbTimeStamp = newTimestamp; +} + Document* PortablePdbWriter::GetCurrentDocument() { return m_currentDocument; diff --git a/src/coreclr/ilasm/portable_pdb.h b/src/coreclr/ilasm/portable_pdb.h index 46bd1f1492076..5145348e527a8 100644 --- a/src/coreclr/ilasm/portable_pdb.h +++ b/src/coreclr/ilasm/portable_pdb.h @@ -46,6 +46,8 @@ class PortablePdbWriter IMetaDataEmit3* GetEmitter(); GUID* GetGuid(); ULONG GetTimestamp(); + void SetGuid(REFGUID newGuid); + void SetTimestamp(const ULONG newTimestamp); Document* GetCurrentDocument(); HRESULT BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef entryPoint); HRESULT DefineDocument(char* name, GUID* language); diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 4c06a0ee58bed..6e7bd04ce7399 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -70,11 +70,17 @@ HRESULT Assembler::InitMetaData() } } - if (m_fGeneratePDB) { m_pPortablePdbWriter = new PortablePdbWriter(); if (FAILED(hr = m_pPortablePdbWriter->Init(m_pDisp))) goto exit; + + if (m_fDeterministic) + { + // Default values for determinism. + m_pPortablePdbWriter->SetGuid(GUID()); + m_pPortablePdbWriter->SetTimestamp(0); + } } //m_Parser = new AsmParse(m_pEmitter); @@ -1544,6 +1550,13 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) hr = Sha256Hash(metaData, metaDataSize, (BYTE*)&mvid, sizeof(GUID)); if (FAILED(hr)) goto exit; m_pInternalEmitForDeterministicMvid->ChangeMvid(mvid); + + if (m_fGeneratePDB) + { + GUID pdbGuid; + // TODO: Sha256Hash contents of pdb. + m_pPortablePdbWriter->SetGuid(pdbGuid); + } } if(bClock) bClock->cFilegenBegin = GetTickCount(); @@ -1633,59 +1646,8 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - NTSTATUS status; - - BCRYPT_ALG_HANDLE algHandle = NULL; - BCRYPT_HASH_HANDLE hashHandle = NULL; - - BYTE hash[32]; // 256 bits - DWORD hashLength = 0; - DWORD resultLength = 0; - status = BCryptOpenAlgorithmProvider(&algHandle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_HASH_REUSABLE_FLAG); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - status = BCryptGetProperty(algHandle, BCRYPT_HASH_LENGTH, (PBYTE)&hashLength, sizeof(hashLength), &resultLength, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - if (hashLength != 32) - { - status = STATUS_NO_MEMORY; - goto cleanup; - } - status = BCryptCreateHash(algHandle, &hashHandle, NULL, 0, NULL, 0, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - - status = BCryptHashData(hashHandle, pSrc, srcSize, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - - status = BCryptFinishHash(hashHandle, hash, hashLength, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - memcpy(pDst, hash, min(hashLength, dstSize)); - status = STATUS_SUCCESS; - -cleanup: - if (NULL != hashHandle) - { - BCryptDestroyHash(hashHandle); - } - if(NULL != algHandle) - { - BCryptCloseAlgorithmProvider(algHandle, 0); - } - return status; + // TODO: Implement sha 256 hash. + return 0; } #ifdef _PREFAST_ From 54147cc515f17bc581a5d91e1f2b0c8af146af0c Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 12:45:08 -0700 Subject: [PATCH 04/69] Add use of BCYRPT for windows-only. Added TODO for non-win32. --- src/coreclr/ilasm/writer.cpp | 58 +++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 6e7bd04ce7399..77ab7e9dbfb02 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1646,8 +1646,64 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - // TODO: Implement sha 256 hash. +#if !defined(_WIN32) + // TODO: Use openssl. return 0; +#else + NTSTATUS status; + + BCRYPT_ALG_HANDLE algHandle = NULL; + BCRYPT_HASH_HANDLE hashHandle = NULL; + + BYTE hash[32]; // 256 bits + DWORD hashLength = 0; + DWORD resultLength = 0; + status = BCryptOpenAlgorithmProvider(&algHandle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_HASH_REUSABLE_FLAG); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + status = BCryptGetProperty(algHandle, BCRYPT_HASH_LENGTH, (PBYTE)&hashLength, sizeof(hashLength), &resultLength, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + if (hashLength != 32) + { + status = STATUS_NO_MEMORY; + goto cleanup; + } + status = BCryptCreateHash(algHandle, &hashHandle, NULL, 0, NULL, 0, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + status = BCryptHashData(hashHandle, pSrc, srcSize, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + status = BCryptFinishHash(hashHandle, hash, hashLength, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + memcpy(pDst, hash, min(hashLength, dstSize)); + status = STATUS_SUCCESS; + +cleanup: + if (NULL != hashHandle) + { + BCryptDestroyHash(hashHandle); + } + if(NULL != algHandle) + { + BCryptCloseAlgorithmProvider(algHandle, 0); + } + return status; +#endif } #ifdef _PREFAST_ From 5516e77bc98239e8553a315db5258cc8b86e7644 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 15:14:03 -0700 Subject: [PATCH 05/69] Preliminary work for deterministic PDB guid --- src/coreclr/ilasm/assem.cpp | 11 +++++++++++ src/coreclr/ilasm/assembler.cpp | 3 --- src/coreclr/ilasm/assembler.h | 1 + src/coreclr/ilasm/writer.cpp | 26 +++++++++++++++++++++----- src/coreclr/inc/metadata.h | 5 +++++ src/coreclr/md/compiler/helper.cpp | 13 +++++++++++++ src/coreclr/md/compiler/regmeta.h | 3 +++ src/coreclr/md/enc/pdbheap.cpp | 11 +++++++++++ src/coreclr/md/inc/pdbheap.h | 1 + 9 files changed, 66 insertions(+), 8 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index aaac49098609c..4232e418eeb72 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -29,6 +29,7 @@ Assembler::Assembler() m_pDisp = NULL; m_pEmitter = NULL; m_pInternalEmitForDeterministicMvid = NULL; + m_pInternalEmitForDeterministicPdbGuid = NULL; m_pImporter = NULL; char* pszFQN = new char[16]; @@ -214,6 +215,11 @@ Assembler::~Assembler() m_pInternalEmitForDeterministicMvid->Release(); m_pInternalEmitForDeterministicMvid = NULL; } + if (m_pInternalEmitForDeterministicPdbGuid != NULL) + { + m_pInternalEmitForDeterministicPdbGuid->Release(); + m_pInternalEmitForDeterministicPdbGuid = NULL; + } if (m_pPortablePdbWriter != NULL) { delete m_pPortablePdbWriter; @@ -240,6 +246,11 @@ BOOL Assembler::Init(BOOL generatePdb) if (FAILED(CreateICeeFileGen(&m_pCeeFileGen))) return FALSE; + if (m_fDeterministic) + { + m_dwCeeFileFlags |= ICEE_CREATE_FILE_DET; + } + if (FAILED(m_pCeeFileGen->CreateCeeFileEx(&m_pCeeFile,(ULONG)m_dwCeeFileFlags))) return FALSE; if (FAILED(m_pCeeFileGen->GetSectionCreate(m_pCeeFile, ".il", sdReadOnly, &m_pILSection))) return FALSE; diff --git a/src/coreclr/ilasm/assembler.cpp b/src/coreclr/ilasm/assembler.cpp index de8d0564c7e78..01a5e1d2e4bfb 100644 --- a/src/coreclr/ilasm/assembler.cpp +++ b/src/coreclr/ilasm/assembler.cpp @@ -2422,12 +2422,9 @@ void Assembler::SetPdbFileName(_In_ __nullterminated char* szName) HRESULT Assembler::SavePdbFile() { HRESULT hr = S_OK; - mdMethodDef entryPoint; if (FAILED(hr = (m_pPortablePdbWriter == NULL ? E_FAIL : S_OK))) goto exit; if (FAILED(hr = (m_pPortablePdbWriter->GetEmitter() == NULL ? E_FAIL : S_OK))) goto exit; - if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit; - if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit; if (FAILED(hr = m_pPortablePdbWriter->GetEmitter()->Save(m_wzPdbFileName, NULL))) goto exit; exit: diff --git a/src/coreclr/ilasm/assembler.h b/src/coreclr/ilasm/assembler.h index 38f1fd0d03295..7e7d7a1479152 100644 --- a/src/coreclr/ilasm/assembler.h +++ b/src/coreclr/ilasm/assembler.h @@ -762,6 +762,7 @@ class Assembler { IMetaDataDispenserEx2 *m_pDisp; IMetaDataEmit3 *m_pEmitter; IMDInternalEmit *m_pInternalEmitForDeterministicMvid; + IMDInternalEmit *m_pInternalEmitForDeterministicPdbGuid; ICeeFileGen *m_pCeeFileGen; IMetaDataImport2 *m_pImporter; // Import interface. HCEEFILE m_pCeeFile; diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 77ab7e9dbfb02..8a21da5f64765 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -80,6 +80,15 @@ HRESULT Assembler::InitMetaData() // Default values for determinism. m_pPortablePdbWriter->SetGuid(GUID()); m_pPortablePdbWriter->SetTimestamp(0); + + hr = m_pPortablePdbWriter->GetEmitter()->QueryInterface(IID_IMDInternalEmit, (void**)&m_pInternalEmitForDeterministicPdbGuid); + + if (FAILED(hr) || (m_pInternalEmitForDeterministicPdbGuid == NULL)) + { + fprintf(stderr, "Unexpected: Failed to query the required PDB GUID determinism interface: %X\n",hr); + hr = E_FAIL; + goto exit; + } } } @@ -1315,8 +1324,6 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr=CreateTLSDirectory())) goto exit; - if (FAILED(hr=CreateDebugDirectory())) goto exit; - if (FAILED(hr=m_pCeeFileGen->SetOutputFileName(m_pCeeFile, pwzOutputFilename))) goto exit; // Reserve a buffer for the meta-data @@ -1540,6 +1547,14 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr)) goto exit; } + if (m_fGeneratePDB) + { + mdMethodDef entryPoint; + + if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit; + if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit; + } + if (m_fDeterministic) { // In deterministic mode, the MVID needs to be stabilized for the metadata scope that was @@ -1553,12 +1568,13 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (m_fGeneratePDB) { - GUID pdbGuid; - // TODO: Sha256Hash contents of pdb. - m_pPortablePdbWriter->SetGuid(pdbGuid); + _ASSERTE(m_pInternalEmitForDeterministicPdbGuid != NULL); + m_pInternalEmitForDeterministicPdbGuid->ComputePdbGuid(Sha256Hash); } } + if (FAILED(hr=CreateDebugDirectory())) goto exit; + if(bClock) bClock->cFilegenBegin = GetTickCount(); // actually output the meta-data if (FAILED(hr=m_pCeeFileGen->EmitMetaDataAt(m_pCeeFile, m_pEmitter, m_pILSection, metaDataOffset, metaData, metaDataSize))) goto exit; diff --git a/src/coreclr/inc/metadata.h b/src/coreclr/inc/metadata.h index 766893bea17b8..ec3fd037bd791 100644 --- a/src/coreclr/inc/metadata.h +++ b/src/coreclr/inc/metadata.h @@ -1118,6 +1118,11 @@ DECLARE_INTERFACE_(IMDInternalEmit, IUnknown) STDMETHOD(SetMDUpdateMode)( ULONG updateMode, ULONG *pPreviousUpdateMode) PURE; +#ifdef FEATURE_METADATA_EMIT_PORTABLE_PDB + STDMETHOD(ComputePdbGuid)( // S_OK or error. + HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)); +#endif // FEATURE_METADATA_EMIT_PORTABLE_PDB + }; // IMDInternalEmit #ifdef FEATURE_METADATA_CUSTOM_DATA_SOURCE diff --git a/src/coreclr/md/compiler/helper.cpp b/src/coreclr/md/compiler/helper.cpp index 955642853d543..40ea43123d414 100644 --- a/src/coreclr/md/compiler/helper.cpp +++ b/src/coreclr/md/compiler/helper.cpp @@ -414,6 +414,19 @@ RegMeta::GetPathSeparator( return S_OK; } // RegMeta::GetPathSeparator + +//******************************************************************************* +// helper to compute the PDB GUID +// +// Implements internal API code:IMDInternalEmit::ComputePdbGuid. +//******************************************************************************* +HRESULT +RegMeta::ComputePdbGuid( // S_OK or error. + HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)) +{ + return m_pStgdb->m_pPdbHeap->ComputeGuid(computeHash); +} + #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB #endif //FEATURE_METADATA_EMIT && FEATURE_METADATA_INTERNAL_APIS diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index 5bfa1a363bb24..cd44c1710a5e2 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -1290,6 +1290,9 @@ class RegMeta : char *path, // [IN] Path string to search. char *separator, // [OUT] Separator used in path string, NULL if none. ULONG *partsCount); // [OUT] Number of parts separated by the separator. + + STDMETHODIMP ComputePdbGuid( // S_OK or error. + HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)); #endif //***************************************************************************** diff --git a/src/coreclr/md/enc/pdbheap.cpp b/src/coreclr/md/enc/pdbheap.cpp index a95cf4ccefd6d..b028e52a5376a 100644 --- a/src/coreclr/md/enc/pdbheap.cpp +++ b/src/coreclr/md/enc/pdbheap.cpp @@ -66,6 +66,17 @@ HRESULT PdbHeap::SetData(PORT_PDB_STREAM* data) return S_OK; } +__checkReturn +HRESULT PdbHeap::ComputeGuid(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)) +{ + _ASSERTE(m_size >= sizeof(PDB_ID)); + + //if (memcpy_s(m_data, m_size, &guid, sizeof(GUID))) + // return E_FAIL; + + return S_OK; +} + __checkReturn HRESULT PdbHeap::SaveToStream(IStream* stream) { diff --git a/src/coreclr/md/inc/pdbheap.h b/src/coreclr/md/inc/pdbheap.h index 92cc3b24494ac..781178e865124 100644 --- a/src/coreclr/md/inc/pdbheap.h +++ b/src/coreclr/md/inc/pdbheap.h @@ -21,6 +21,7 @@ class PdbHeap ~PdbHeap(); __checkReturn HRESULT SetData(PORT_PDB_STREAM* data); + __checkReturn HRESULT ComputeGuid(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)); __checkReturn HRESULT SaveToStream(IStream* stream); BOOL IsEmpty(); ULONG GetSize(); From d6a16faf2c97e03a1d78447f79e4e7fec0c646de Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 15:35:33 -0700 Subject: [PATCH 06/69] Fixed ordering problem --- src/coreclr/ilasm/writer.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 8a21da5f64765..8e5936195d54d 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1324,6 +1324,22 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr=CreateTLSDirectory())) goto exit; + if (m_fGeneratePDB) + { + mdMethodDef entryPoint; + + if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit; + if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit; + + if (m_fDeterministic) + { + _ASSERTE(m_pInternalEmitForDeterministicPdbGuid != NULL); + m_pInternalEmitForDeterministicPdbGuid->ComputePdbGuid(Sha256Hash); + } + } + + if (FAILED(hr=CreateDebugDirectory())) goto exit; + if (FAILED(hr=m_pCeeFileGen->SetOutputFileName(m_pCeeFile, pwzOutputFilename))) goto exit; // Reserve a buffer for the meta-data @@ -1547,14 +1563,6 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr)) goto exit; } - if (m_fGeneratePDB) - { - mdMethodDef entryPoint; - - if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit; - if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit; - } - if (m_fDeterministic) { // In deterministic mode, the MVID needs to be stabilized for the metadata scope that was @@ -1565,16 +1573,8 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) hr = Sha256Hash(metaData, metaDataSize, (BYTE*)&mvid, sizeof(GUID)); if (FAILED(hr)) goto exit; m_pInternalEmitForDeterministicMvid->ChangeMvid(mvid); - - if (m_fGeneratePDB) - { - _ASSERTE(m_pInternalEmitForDeterministicPdbGuid != NULL); - m_pInternalEmitForDeterministicPdbGuid->ComputePdbGuid(Sha256Hash); - } } - if (FAILED(hr=CreateDebugDirectory())) goto exit; - if(bClock) bClock->cFilegenBegin = GetTickCount(); // actually output the meta-data if (FAILED(hr=m_pCeeFileGen->EmitMetaDataAt(m_pCeeFile, m_pEmitter, m_pILSection, metaDataOffset, metaData, metaDataSize))) goto exit; From f2e4d2c01de88330cb9864069cdb5bb92074e615 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Apr 2023 16:26:45 -0700 Subject: [PATCH 07/69] Computing checksum and using result for the PDB guid --- src/coreclr/ilasm/assem.cpp | 8 ++++---- src/coreclr/ilasm/assembler.h | 2 +- src/coreclr/ilasm/writer.cpp | 14 ++++++++++---- src/coreclr/inc/metadata.h | 8 ++++++-- src/coreclr/md/compiler/helper.cpp | 21 +++++++++++++++++---- src/coreclr/md/compiler/regmeta.h | 8 ++++++-- src/coreclr/md/enc/pdbheap.cpp | 14 +++++++++++--- src/coreclr/md/inc/pdbheap.h | 3 ++- 8 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index 4232e418eeb72..572c34253865c 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -29,7 +29,7 @@ Assembler::Assembler() m_pDisp = NULL; m_pEmitter = NULL; m_pInternalEmitForDeterministicMvid = NULL; - m_pInternalEmitForDeterministicPdbGuid = NULL; + m_pInternalEmitForDeterministicPdb = NULL; m_pImporter = NULL; char* pszFQN = new char[16]; @@ -215,10 +215,10 @@ Assembler::~Assembler() m_pInternalEmitForDeterministicMvid->Release(); m_pInternalEmitForDeterministicMvid = NULL; } - if (m_pInternalEmitForDeterministicPdbGuid != NULL) + if (m_pInternalEmitForDeterministicPdb != NULL) { - m_pInternalEmitForDeterministicPdbGuid->Release(); - m_pInternalEmitForDeterministicPdbGuid = NULL; + m_pInternalEmitForDeterministicPdb->Release(); + m_pInternalEmitForDeterministicPdb = NULL; } if (m_pPortablePdbWriter != NULL) { diff --git a/src/coreclr/ilasm/assembler.h b/src/coreclr/ilasm/assembler.h index 7e7d7a1479152..efa3918769638 100644 --- a/src/coreclr/ilasm/assembler.h +++ b/src/coreclr/ilasm/assembler.h @@ -762,7 +762,7 @@ class Assembler { IMetaDataDispenserEx2 *m_pDisp; IMetaDataEmit3 *m_pEmitter; IMDInternalEmit *m_pInternalEmitForDeterministicMvid; - IMDInternalEmit *m_pInternalEmitForDeterministicPdbGuid; + IMDInternalEmit *m_pInternalEmitForDeterministicPdb; ICeeFileGen *m_pCeeFileGen; IMetaDataImport2 *m_pImporter; // Import interface. HCEEFILE m_pCeeFile; diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 8e5936195d54d..1d03c85e58a03 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -81,9 +81,9 @@ HRESULT Assembler::InitMetaData() m_pPortablePdbWriter->SetGuid(GUID()); m_pPortablePdbWriter->SetTimestamp(0); - hr = m_pPortablePdbWriter->GetEmitter()->QueryInterface(IID_IMDInternalEmit, (void**)&m_pInternalEmitForDeterministicPdbGuid); + hr = m_pPortablePdbWriter->GetEmitter()->QueryInterface(IID_IMDInternalEmit, (void**)&m_pInternalEmitForDeterministicPdb); - if (FAILED(hr) || (m_pInternalEmitForDeterministicPdbGuid == NULL)) + if (FAILED(hr) || (m_pInternalEmitForDeterministicPdb == NULL)) { fprintf(stderr, "Unexpected: Failed to query the required PDB GUID determinism interface: %X\n",hr); hr = E_FAIL; @@ -1333,8 +1333,14 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (m_fDeterministic) { - _ASSERTE(m_pInternalEmitForDeterministicPdbGuid != NULL); - m_pInternalEmitForDeterministicPdbGuid->ComputePdbGuid(Sha256Hash); + _ASSERTE(m_pInternalEmitForDeterministicPdb != NULL); + + BYTE pdbChecksum[32]; + if (FAILED(hr = m_pInternalEmitForDeterministicPdb->ComputeSha256PdbChecksum(Sha256Hash, pdbChecksum))) goto exit; + + GUID pdbGuid = *((GUID*)&pdbChecksum); + if (FAILED(hr = m_pInternalEmitForDeterministicPdb->ChangePdbGuid(pdbGuid))) goto exit; + m_pPortablePdbWriter->SetGuid(pdbGuid); } } diff --git a/src/coreclr/inc/metadata.h b/src/coreclr/inc/metadata.h index ec3fd037bd791..2b0c30002d773 100644 --- a/src/coreclr/inc/metadata.h +++ b/src/coreclr/inc/metadata.h @@ -1119,8 +1119,12 @@ DECLARE_INTERFACE_(IMDInternalEmit, IUnknown) ULONG updateMode, ULONG *pPreviousUpdateMode) PURE; #ifdef FEATURE_METADATA_EMIT_PORTABLE_PDB - STDMETHOD(ComputePdbGuid)( // S_OK or error. - HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)); + STDMETHOD(ComputeSha256PdbChecksum)( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), + BYTE (&checksum)[32]); + + STDMETHOD(ChangePdbGuid)( // S_OK or error. + REFGUID newGuid); // GUID to use as the PDB GUID #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB }; // IMDInternalEmit diff --git a/src/coreclr/md/compiler/helper.cpp b/src/coreclr/md/compiler/helper.cpp index 40ea43123d414..c3a7f8e7611f6 100644 --- a/src/coreclr/md/compiler/helper.cpp +++ b/src/coreclr/md/compiler/helper.cpp @@ -416,15 +416,28 @@ RegMeta::GetPathSeparator( } // RegMeta::GetPathSeparator //******************************************************************************* -// helper to compute the PDB GUID +// helper to compute the checksum for the PDB +// +// Implements internal API code:IMDInternalEmit::ComputeSha256Checksum. +//******************************************************************************* +HRESULT +RegMeta::ComputeSha256PdbChecksum( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), + BYTE (&checksum)[32]) +{ + return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); +} + +//******************************************************************************* +// helper to set the PDB GUID // // Implements internal API code:IMDInternalEmit::ComputePdbGuid. //******************************************************************************* HRESULT -RegMeta::ComputePdbGuid( // S_OK or error. - HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)) +RegMeta::ChangePdbGuid( // S_OK or error. + REFGUID newGuid) { - return m_pStgdb->m_pPdbHeap->ComputeGuid(computeHash); + return m_pStgdb->m_pPdbHeap->SetDataGuid(newGuid); } #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index cd44c1710a5e2..802f41db90364 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -1291,8 +1291,12 @@ class RegMeta : char *separator, // [OUT] Separator used in path string, NULL if none. ULONG *partsCount); // [OUT] Number of parts separated by the separator. - STDMETHODIMP ComputePdbGuid( // S_OK or error. - HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)); + STDMETHODIMP ComputeSha256PdbChecksum( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), + BYTE (&checksum)[32]); + + STDMETHODIMP ChangePdbGuid( // S_OK or error. + REFGUID newGuid); // GUID to use as the PDB GUID #endif //***************************************************************************** diff --git a/src/coreclr/md/enc/pdbheap.cpp b/src/coreclr/md/enc/pdbheap.cpp index b028e52a5376a..d402cddc0a731 100644 --- a/src/coreclr/md/enc/pdbheap.cpp +++ b/src/coreclr/md/enc/pdbheap.cpp @@ -66,17 +66,25 @@ HRESULT PdbHeap::SetData(PORT_PDB_STREAM* data) return S_OK; } + __checkReturn -HRESULT PdbHeap::ComputeGuid(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)) +HRESULT PdbHeap::SetDataGuid(REFGUID newGuid) { _ASSERTE(m_size >= sizeof(PDB_ID)); - //if (memcpy_s(m_data, m_size, &guid, sizeof(GUID))) - // return E_FAIL; + if (memcpy_s(m_data, m_size, &newGuid, sizeof(GUID))) + return E_FAIL; return S_OK; } +__checkReturn +HRESULT PdbHeap::ComputeSha256Checksum(HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]) +{ + _ASSERTE(m_size >= sizeof(PDB_ID)); + return computeSha256(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); +} + __checkReturn HRESULT PdbHeap::SaveToStream(IStream* stream) { diff --git a/src/coreclr/md/inc/pdbheap.h b/src/coreclr/md/inc/pdbheap.h index 781178e865124..88bae893f86d7 100644 --- a/src/coreclr/md/inc/pdbheap.h +++ b/src/coreclr/md/inc/pdbheap.h @@ -21,7 +21,8 @@ class PdbHeap ~PdbHeap(); __checkReturn HRESULT SetData(PORT_PDB_STREAM* data); - __checkReturn HRESULT ComputeGuid(HRESULT (*computeHash)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize)); + __checkReturn HRESULT SetDataGuid(REFGUID newGuid); + __checkReturn HRESULT ComputeSha256Checksum(HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]); __checkReturn HRESULT SaveToStream(IStream* stream); BOOL IsEmpty(); ULONG GetSize(); From 89df2d8b69fed84b99b11e4d5dd4b9294c5f1e08 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Apr 2023 11:00:47 -0700 Subject: [PATCH 08/69] Refactoring apis --- src/coreclr/ilasm/assem.cpp | 6 ----- src/coreclr/ilasm/assembler.h | 1 - src/coreclr/ilasm/portable_pdb.cpp | 11 +++++++++ src/coreclr/ilasm/portable_pdb.h | 2 ++ src/coreclr/ilasm/writer.cpp | 24 +++++------------- src/coreclr/inc/metadata.h | 10 -------- src/coreclr/md/compiler/emit.cpp | 38 +++++++++++++++++++++++++++++ src/coreclr/md/compiler/helper.cpp | 25 ------------------- src/coreclr/md/compiler/regmeta.h | 14 +++++------ src/coreclr/md/inc/portablepdbmdi.h | 7 ++++++ 10 files changed, 71 insertions(+), 67 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index 572c34253865c..85aa5c12ac36e 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -29,7 +29,6 @@ Assembler::Assembler() m_pDisp = NULL; m_pEmitter = NULL; m_pInternalEmitForDeterministicMvid = NULL; - m_pInternalEmitForDeterministicPdb = NULL; m_pImporter = NULL; char* pszFQN = new char[16]; @@ -215,11 +214,6 @@ Assembler::~Assembler() m_pInternalEmitForDeterministicMvid->Release(); m_pInternalEmitForDeterministicMvid = NULL; } - if (m_pInternalEmitForDeterministicPdb != NULL) - { - m_pInternalEmitForDeterministicPdb->Release(); - m_pInternalEmitForDeterministicPdb = NULL; - } if (m_pPortablePdbWriter != NULL) { delete m_pPortablePdbWriter; diff --git a/src/coreclr/ilasm/assembler.h b/src/coreclr/ilasm/assembler.h index efa3918769638..38f1fd0d03295 100644 --- a/src/coreclr/ilasm/assembler.h +++ b/src/coreclr/ilasm/assembler.h @@ -762,7 +762,6 @@ class Assembler { IMetaDataDispenserEx2 *m_pDisp; IMetaDataEmit3 *m_pEmitter; IMDInternalEmit *m_pInternalEmitForDeterministicMvid; - IMDInternalEmit *m_pInternalEmitForDeterministicPdb; ICeeFileGen *m_pCeeFileGen; IMetaDataImport2 *m_pImporter; // Import interface. HCEEFILE m_pCeeFile; diff --git a/src/coreclr/ilasm/portable_pdb.cpp b/src/coreclr/ilasm/portable_pdb.cpp index 8e52bd2ac8161..65df6863f3552 100644 --- a/src/coreclr/ilasm/portable_pdb.cpp +++ b/src/coreclr/ilasm/portable_pdb.cpp @@ -153,6 +153,17 @@ HRESULT PortablePdbWriter::BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef return hr; } +HRESULT PortablePdbWriter::ComputeSha256PdbStreamChecksum(BYTE(&checksum)[32]) +{ + return m_pdbEmitter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum); +} + +HRESULT PortablePdbWriter::ChangePdbStreamGuid(REFGUID newGuid) +{ + m_pdbStream.id.pdbGuid = newGuid; + return m_pdbEmitter->ChangePdbStreamGuid(newGuid); +} + HRESULT PortablePdbWriter::DefineDocument(char* name, GUID* language) { HRESULT hr = S_OK; diff --git a/src/coreclr/ilasm/portable_pdb.h b/src/coreclr/ilasm/portable_pdb.h index 5145348e527a8..1bca066ceb104 100644 --- a/src/coreclr/ilasm/portable_pdb.h +++ b/src/coreclr/ilasm/portable_pdb.h @@ -50,6 +50,8 @@ class PortablePdbWriter void SetTimestamp(const ULONG newTimestamp); Document* GetCurrentDocument(); HRESULT BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef entryPoint); + HRESULT ComputeSha256PdbStreamChecksum(BYTE (&checksum)[32]); + HRESULT ChangePdbStreamGuid(REFGUID newGuid); HRESULT DefineDocument(char* name, GUID* language); HRESULT DefineSequencePoints(Method* method); HRESULT DefineLocalScope(Method* method); diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 1d03c85e58a03..9017b016f3b46 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -80,15 +80,6 @@ HRESULT Assembler::InitMetaData() // Default values for determinism. m_pPortablePdbWriter->SetGuid(GUID()); m_pPortablePdbWriter->SetTimestamp(0); - - hr = m_pPortablePdbWriter->GetEmitter()->QueryInterface(IID_IMDInternalEmit, (void**)&m_pInternalEmitForDeterministicPdb); - - if (FAILED(hr) || (m_pInternalEmitForDeterministicPdb == NULL)) - { - fprintf(stderr, "Unexpected: Failed to query the required PDB GUID determinism interface: %X\n",hr); - hr = E_FAIL; - goto exit; - } } } @@ -1331,16 +1322,13 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit; if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit; + BYTE pdbChecksum[32]; + if (FAILED(hr = m_pPortablePdbWriter->ComputeSha256PdbStreamChecksum(pdbChecksum))) goto exit; + if (m_fDeterministic) { - _ASSERTE(m_pInternalEmitForDeterministicPdb != NULL); - - BYTE pdbChecksum[32]; - if (FAILED(hr = m_pInternalEmitForDeterministicPdb->ComputeSha256PdbChecksum(Sha256Hash, pdbChecksum))) goto exit; - GUID pdbGuid = *((GUID*)&pdbChecksum); - if (FAILED(hr = m_pInternalEmitForDeterministicPdb->ChangePdbGuid(pdbGuid))) goto exit; - m_pPortablePdbWriter->SetGuid(pdbGuid); + if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(pdbGuid))) goto exit; } } @@ -1669,8 +1657,8 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { #if !defined(_WIN32) - // TODO: Use openssl. - return 0; + // TODO: Use openssl? + return S_OK; #else NTSTATUS status; diff --git a/src/coreclr/inc/metadata.h b/src/coreclr/inc/metadata.h index 2b0c30002d773..ad0adc027b066 100644 --- a/src/coreclr/inc/metadata.h +++ b/src/coreclr/inc/metadata.h @@ -1117,16 +1117,6 @@ DECLARE_INTERFACE_(IMDInternalEmit, IUnknown) STDMETHOD(SetMDUpdateMode)( ULONG updateMode, ULONG *pPreviousUpdateMode) PURE; - -#ifdef FEATURE_METADATA_EMIT_PORTABLE_PDB - STDMETHOD(ComputeSha256PdbChecksum)( // S_OK or error. - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), - BYTE (&checksum)[32]); - - STDMETHOD(ChangePdbGuid)( // S_OK or error. - REFGUID newGuid); // GUID to use as the PDB GUID -#endif // FEATURE_METADATA_EMIT_PORTABLE_PDB - }; // IMDInternalEmit #ifdef FEATURE_METADATA_CUSTOM_DATA_SOURCE diff --git a/src/coreclr/md/compiler/emit.cpp b/src/coreclr/md/compiler/emit.cpp index bdfa534e075c9..f56d2bda9043a 100644 --- a/src/coreclr/md/compiler/emit.cpp +++ b/src/coreclr/md/compiler/emit.cpp @@ -2105,6 +2105,44 @@ STDMETHODIMP RegMeta::DefineLocalVariable( // S_OK or error. return hr; #endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER } // RegMeta::DefineLocalVariable + +//******************************************************************************* +// ComputeSha256PdbStreamChecksum +//******************************************************************************* +STDMETHODIMP RegMeta::ComputeSha256PdbStreamChecksum( + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), + BYTE (&checksum)[32]) +{ +#ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER + return E_NOTIMPL; +#else //!FEATURE_METADATA_EMIT_IN_DEBUGGER + HRESULT hr = S_OK; + + hr = m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); + +ErrExit: + return hr; +#endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER +} + +//******************************************************************************* +// ChangePdbStreamGuid +//******************************************************************************* +STDMETHODIMP RegMeta::ChangePdbStreamGuid( + REFGUID newGuid) +{ +#ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER + return E_NOTIMPL; +#else //!FEATURE_METADATA_EMIT_IN_DEBUGGER + HRESULT hr = S_OK; + + hr = m_pStgdb->m_pPdbHeap->SetDataGuid(newGuid); + +ErrExit: + return hr; +#endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER +} + #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB //***************************************************************************** diff --git a/src/coreclr/md/compiler/helper.cpp b/src/coreclr/md/compiler/helper.cpp index c3a7f8e7611f6..c7d0f2d305c2e 100644 --- a/src/coreclr/md/compiler/helper.cpp +++ b/src/coreclr/md/compiler/helper.cpp @@ -415,31 +415,6 @@ RegMeta::GetPathSeparator( return S_OK; } // RegMeta::GetPathSeparator -//******************************************************************************* -// helper to compute the checksum for the PDB -// -// Implements internal API code:IMDInternalEmit::ComputeSha256Checksum. -//******************************************************************************* -HRESULT -RegMeta::ComputeSha256PdbChecksum( // S_OK or error. - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), - BYTE (&checksum)[32]) -{ - return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); -} - -//******************************************************************************* -// helper to set the PDB GUID -// -// Implements internal API code:IMDInternalEmit::ComputePdbGuid. -//******************************************************************************* -HRESULT -RegMeta::ChangePdbGuid( // S_OK or error. - REFGUID newGuid) -{ - return m_pStgdb->m_pPdbHeap->SetDataGuid(newGuid); -} - #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB #endif //FEATURE_METADATA_EMIT && FEATURE_METADATA_INTERNAL_APIS diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index 802f41db90364..65621e87d082a 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -1132,6 +1132,13 @@ class RegMeta : USHORT index, // [IN] Variable index (slot). char *name, // [IN] Variable name. mdLocalVariable *locVarToken); // [OUT] Token of the defined variable. + + STDMETHODIMP ComputeSha256PdbStreamChecksum( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] + BYTE (&checksum)[32]); // [OUT] 256-bit Pdb checksum + + STDMETHODIMP ChangePdbStreamGuid( // S_OK or error. + REFGUID newGuid); // [IN] GUID to use as the PDB GUID #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB //***************************************************************************** @@ -1290,13 +1297,6 @@ class RegMeta : char *path, // [IN] Path string to search. char *separator, // [OUT] Separator used in path string, NULL if none. ULONG *partsCount); // [OUT] Number of parts separated by the separator. - - STDMETHODIMP ComputeSha256PdbChecksum( // S_OK or error. - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), - BYTE (&checksum)[32]); - - STDMETHODIMP ChangePdbGuid( // S_OK or error. - REFGUID newGuid); // GUID to use as the PDB GUID #endif //***************************************************************************** diff --git a/src/coreclr/md/inc/portablepdbmdi.h b/src/coreclr/md/inc/portablepdbmdi.h index d1785bca193b3..62cac609f2c60 100644 --- a/src/coreclr/md/inc/portablepdbmdi.h +++ b/src/coreclr/md/inc/portablepdbmdi.h @@ -69,6 +69,13 @@ DECLARE_INTERFACE_(IMetaDataEmit3, IMetaDataEmit2) USHORT index, // [IN] Variable index (slot). char *name, // [IN] Variable name. mdLocalVariable *locVarToken) PURE; // [OUT] Token of the defined variable. + + STDMETHOD(ComputeSha256PdbStreamChecksum)( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] + BYTE (&checksum)[32]) PURE; // [OUT] 256-bit Pdb checksum + + STDMETHOD(ChangePdbStreamGuid)( // S_OK or error. + REFGUID newGuid) PURE; // [IN] GUID to use as the PDB GUID }; //------------------------------------- From 7374450fd14bd1f780ee43d6147e5f7dde479861 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Apr 2023 15:10:12 -0700 Subject: [PATCH 09/69] Added PdbChecksum and Deterministic Debug Directory entries --- src/coreclr/ilasm/assembler.h | 2 +- src/coreclr/ilasm/writer.cpp | 301 ++++++++++++++++++++++--------- src/coreclr/md/compiler/emit.cpp | 14 +- 3 files changed, 217 insertions(+), 100 deletions(-) diff --git a/src/coreclr/ilasm/assembler.h b/src/coreclr/ilasm/assembler.h index 38f1fd0d03295..456af751a926a 100644 --- a/src/coreclr/ilasm/assembler.h +++ b/src/coreclr/ilasm/assembler.h @@ -847,7 +847,7 @@ class Assembler { BOOL EmitClass(Class *pClass); HRESULT CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename); HRESULT CreateTLSDirectory(); - HRESULT CreateDebugDirectory(); + HRESULT CreateDebugDirectory(BYTE(&pdbChecksum)[32]); HRESULT InitMetaData(); Class *FindCreateClass(_In_ __nullterminated const char *pszFQN); BOOL EmitFieldRef(_In_z_ char *pszArg, int opcode); diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 9017b016f3b46..5a328d5e5a475 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -226,126 +226,253 @@ HRESULT Assembler::CreateTLSDirectory() { return(hr); } -HRESULT Assembler::CreateDebugDirectory() +HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) { - HRESULT hr = S_OK; - HCEESECTION sec = m_pILSection; - BYTE *de; - ULONG deOffset; - // Only emit this if we're also emitting debug info. - if (!m_fGeneratePDB) - return S_OK; + _ASSERTE(m_fGeneratePDB); - IMAGE_DEBUG_DIRECTORY debugDirIDD; - struct Param + struct DebugDirectoryEntry { - DWORD debugDirDataSize; - BYTE *debugDirData; - } param; - param.debugDirData = NULL; + IMAGE_DEBUG_DIRECTORY debugDirIDD; + DWORD debugDirDataSize; + BYTE* debugDirData; + }; + + // Arbitrary amount; should not need this many. + const int maxEntries = 8; + + DebugDirectoryEntry entries[maxEntries]; + int numEntries = 0; + + auto addEntry = [&]( + DWORD characteristics, + DWORD timeDateStamp, + WORD majorVersion, + WORD minorVersion, + DWORD type, + DWORD sizeOfData, + BYTE* data){ + _ASSERTE(numEntries >= 0); + _ASSERTE(numEntries < maxEntries); + + HRESULT hr = S_OK; + + IMAGE_DEBUG_DIRECTORY debugDirIDD; + struct Param + { + DWORD debugDirDataSize; + BYTE* debugDirData; + } param; + param.debugDirData = NULL; + + debugDirIDD.Characteristics = characteristics; + debugDirIDD.TimeDateStamp = timeDateStamp; + debugDirIDD.MajorVersion = majorVersion; + debugDirIDD.MinorVersion = minorVersion; + debugDirIDD.Type = type; + debugDirIDD.SizeOfData = sizeOfData; + debugDirIDD.AddressOfRawData = 0; // will be updated later + debugDirIDD.PointerToRawData = 0; // will be updated later + + param.debugDirDataSize = sizeOfData; + + if ((sizeOfData > 0) && (data != NULL)) + { + // Make some room for the data. + PAL_TRY(Param*, pParam, ¶m) { + pParam->debugDirData = new BYTE[pParam->debugDirDataSize]; + } PAL_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { + hr = E_FAIL; + } PAL_ENDTRY + + if (FAILED(hr)) return hr; + } + + param.debugDirData = data; + + DebugDirectoryEntry entry = { 0 }; + entry.debugDirIDD = debugDirIDD; + entry.debugDirDataSize = param.debugDirDataSize; + entry.debugDirData = param.debugDirData; + entries[numEntries] = entry; + + numEntries++; + return S_OK; + }; + HRESULT hr = S_OK; + + /* BEGIN CODEVIEW */ // get module ID DWORD rsds = VAL32(0x53445352); DWORD pdbAge = VAL32(0x1); GUID pdbGuid = *m_pPortablePdbWriter->GetGuid(); SwapGuid(&pdbGuid); - DWORD len = sizeof(rsds) + sizeof(GUID) + sizeof(pdbAge) + (DWORD)strlen(m_szPdbFileName) + 1; - BYTE* dbgDirData = new BYTE[len]; - - DWORD offset = 0; - memcpy_s(dbgDirData + offset, len, &rsds, sizeof(rsds)); // RSDS - offset += sizeof(rsds); - memcpy_s(dbgDirData + offset, len, &pdbGuid, sizeof(GUID)); // PDB GUID - offset += sizeof(GUID); - memcpy_s(dbgDirData + offset, len, &pdbAge, sizeof(pdbAge)); // PDB AGE - offset += sizeof(pdbAge); - memcpy_s(dbgDirData + offset, len, m_szPdbFileName, strlen(m_szPdbFileName) + 1); // PDB PATH - - debugDirIDD.Characteristics = 0; - debugDirIDD.TimeDateStamp = VAL32(m_pPortablePdbWriter->GetTimestamp()); - debugDirIDD.MajorVersion = VAL16(0x100); - debugDirIDD.MinorVersion = VAL16(0x504d); - debugDirIDD.Type = VAL32(IMAGE_DEBUG_TYPE_CODEVIEW); - debugDirIDD.SizeOfData = VAL32(len); - debugDirIDD.AddressOfRawData = 0; // will be updated bellow - debugDirIDD.PointerToRawData = 0; // will be updated bellow - - param.debugDirDataSize = len; - - // Make some room for the data. - PAL_TRY(Param*, pParam, ¶m) { - pParam->debugDirData = new BYTE[pParam->debugDirDataSize]; - } PAL_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { - hr = E_FAIL; - } PAL_ENDTRY + DWORD codeViewSize = sizeof(rsds) + sizeof(GUID) + sizeof(pdbAge) + (DWORD)strlen(m_szPdbFileName) + 1; + BYTE* codeViewData = new BYTE[codeViewSize]; + + DWORD codeViewOffset = 0; + memcpy_s(codeViewData + codeViewOffset, codeViewSize, &rsds, sizeof(rsds)); // RSDS + codeViewOffset += sizeof(rsds); + memcpy_s(codeViewData + codeViewOffset, codeViewSize, &pdbGuid, sizeof(GUID)); // PDB GUID + codeViewOffset += sizeof(GUID); + memcpy_s(codeViewData + codeViewOffset, codeViewSize, &pdbAge, sizeof(pdbAge)); // PDB AGE + codeViewOffset += sizeof(pdbAge); + memcpy_s(codeViewData + codeViewOffset, codeViewSize, m_szPdbFileName, strlen(m_szPdbFileName) + 1); // PDB PATH + /* END CODEVIEW */ + + /* BEGIN PDB CHECKSUM */ + _ASSERTE(sizeof(pdbChecksum) == 32); + + // Algorithm name is case sensitive. + const char* algoName = "SHA256"; + DWORD pdbChecksumSize = (DWORD)strlen(algoName) + 1 + sizeof(pdbChecksum); + BYTE* pdbChecksumData = new BYTE[pdbChecksumSize]; + + DWORD pdbChecksumOffset = 0; + memcpy_s(pdbChecksumData + pdbChecksumOffset, pdbChecksumSize, algoName, strlen(algoName)); // AlgorithmName + pdbChecksumOffset += (DWORD)strlen(algoName) + 1; + memcpy_s(pdbChecksumData + pdbChecksumOffset, pdbChecksumSize, &pdbChecksum, sizeof(pdbChecksum)); // Checksum + /* END PDB CHECKSUM */ + + // CodeView Entry + hr = + addEntry( + /* characteristics */ VAL32(0), + /* timeDateStamp */ VAL32(m_pPortablePdbWriter->GetTimestamp()), + /* majorVersion */ VAL16(0x100), + /* minorVersion */ VAL16(0x504d), + /* type */ VAL32(IMAGE_DEBUG_TYPE_CODEVIEW), + /* sizeOfData */ VAL32(codeViewSize), + /* data */ codeViewData + ); + if (FAILED(hr)) + goto Exit; + + // Pdb Checksum Entry + hr = + addEntry( + /* characteristics */ VAL32(0), + /* timeDateStamp */ VAL32(0), + /* majorVersion */ VAL16(1), + /* minorVersion */ VAL16(0), + /* type */ VAL32(/* PDB Checksum Debug Directory Entry */ 19), + /* sizeOfData */ VAL32(pdbChecksumSize), + /* data */ pdbChecksumData + ); + if (FAILED(hr)) + goto Exit; - if (FAILED(hr)) return hr; + if (m_fDeterministic) + { + // Deterministic Entry + hr = + addEntry( + /* characteristics */ VAL32(0), + /* timeDateStamp */ VAL32(0), + /* majorVersion */ VAL16(0), + /* minorVersion */ VAL16(0), + /* type */ VAL32(/* Deterministic Debug Directory Entry */ 16), + /* sizeOfData */ VAL32(0), + /* data */ NULL + ); + if (FAILED(hr)) + goto Exit; + } - param.debugDirData = dbgDirData; + HCEESECTION sec = m_pILSection; + BYTE *de; + ULONG deOffset; + + ULONG totalDataSize = 0; + for (int i = 0; i < numEntries; i++) + { + totalDataSize += entries[i].debugDirDataSize; + } + ULONG totalEntrySize = (sizeof(IMAGE_DEBUG_DIRECTORY) * numEntries); + ULONG totalSize = (totalEntrySize + totalDataSize); // Grab memory in the section for our stuff. // Note that UpdateResource doesn't work correctly if the debug directory is // in the data section. So instead we put it in the text section (same as // cs compiler). if (FAILED(hr = m_pCeeFileGen->GetSectionBlock(sec, - sizeof(debugDirIDD) + - param.debugDirDataSize, + totalSize, 4, (void**) &de))) - goto ErrExit; + goto Exit; // Where did we get that memory? if (FAILED(hr = m_pCeeFileGen->GetSectionDataLen(sec, &deOffset))) - goto ErrExit; - - deOffset -= (sizeof(debugDirIDD) + param.debugDirDataSize); - - // Setup a reloc so that the address of the raw - // data is setup correctly. - debugDirIDD.PointerToRawData = VAL32(deOffset + sizeof(debugDirIDD)); - - if (FAILED(hr = m_pCeeFileGen->AddSectionReloc( - sec, - deOffset + - offsetof(IMAGE_DEBUG_DIRECTORY, - PointerToRawData), - sec, srRelocFilePos))) - goto ErrExit; - - debugDirIDD.AddressOfRawData = VAL32(deOffset + sizeof(debugDirIDD)); - - if (FAILED(hr = m_pCeeFileGen->AddSectionReloc( - sec, - deOffset + - offsetof(IMAGE_DEBUG_DIRECTORY, - AddressOfRawData), - sec, srRelocAbsolute))) - goto ErrExit; + goto Exit; + + deOffset -= totalSize; + // Emit the directory entry. if (FAILED(hr = m_pCeeFileGen->SetDirectoryEntry(m_pCeeFile, sec, IMAGE_DIRECTORY_ENTRY_DEBUG, - sizeof(debugDirIDD), + totalEntrySize, deOffset))) - goto ErrExit; + goto Exit; - // Copy the debug directory into the section. - memcpy(de, &debugDirIDD, sizeof(debugDirIDD)); - memcpy(de + sizeof(debugDirIDD), param.debugDirData, - param.debugDirDataSize); + ULONG rawDataOffset = deOffset + totalEntrySize; - if (param.debugDirData) + ULONG dataOffset = 0; + for (int i = 0; i < numEntries; i++) { - delete [] param.debugDirData; + DebugDirectoryEntry* entry = &entries[i]; + + ULONG imageOffset = (i * sizeof(IMAGE_DEBUG_DIRECTORY)); + + if ((entry->debugDirDataSize > 0) && (entry->debugDirData != NULL)) + { + // Setup a reloc so that the address of the raw + // data is setup correctly. + entry->debugDirIDD.PointerToRawData = VAL32(rawDataOffset + dataOffset); + entry->debugDirIDD.AddressOfRawData = VAL32(rawDataOffset + dataOffset); + + dataOffset += entry->debugDirDataSize; + + if (FAILED(hr = m_pCeeFileGen->AddSectionReloc( + sec, + deOffset + imageOffset + + offsetof(IMAGE_DEBUG_DIRECTORY, + PointerToRawData), + sec, srRelocFilePos))) + goto Exit; + + if (FAILED(hr = m_pCeeFileGen->AddSectionReloc( + sec, + deOffset + imageOffset + + offsetof(IMAGE_DEBUG_DIRECTORY, + AddressOfRawData), + sec, srRelocAbsolute))) + goto Exit; + } + + // Copy the debug directory into the section. + memcpy(de + imageOffset, &entry->debugDirIDD, sizeof(IMAGE_DEBUG_DIRECTORY)); } - return S_OK; -ErrExit: - if (param.debugDirData) + dataOffset = 0; + for (int i = 0; i < numEntries; i++) + { + DebugDirectoryEntry entry = entries[i]; + + memcpy(de + totalEntrySize + dataOffset, entry.debugDirData, entry.debugDirDataSize); + dataOffset += entry.debugDirDataSize; + } + +Exit: + if (codeViewData) + { + delete [] codeViewData; + } + if (pdbChecksumData) { - delete [] param.debugDirData; + delete [] pdbChecksumData; } return hr; } @@ -1330,9 +1457,9 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) GUID pdbGuid = *((GUID*)&pdbChecksum); if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(pdbGuid))) goto exit; } - } - if (FAILED(hr=CreateDebugDirectory())) goto exit; + if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; + } if (FAILED(hr=m_pCeeFileGen->SetOutputFileName(m_pCeeFile, pwzOutputFilename))) goto exit; diff --git a/src/coreclr/md/compiler/emit.cpp b/src/coreclr/md/compiler/emit.cpp index f56d2bda9043a..de3b67c4bf86d 100644 --- a/src/coreclr/md/compiler/emit.cpp +++ b/src/coreclr/md/compiler/emit.cpp @@ -2116,12 +2116,7 @@ STDMETHODIMP RegMeta::ComputeSha256PdbStreamChecksum( #ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER return E_NOTIMPL; #else //!FEATURE_METADATA_EMIT_IN_DEBUGGER - HRESULT hr = S_OK; - - hr = m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); - -ErrExit: - return hr; + return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); #endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER } @@ -2134,12 +2129,7 @@ STDMETHODIMP RegMeta::ChangePdbStreamGuid( #ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER return E_NOTIMPL; #else //!FEATURE_METADATA_EMIT_IN_DEBUGGER - HRESULT hr = S_OK; - - hr = m_pStgdb->m_pPdbHeap->SetDataGuid(newGuid); - -ErrExit: - return hr; + return m_pStgdb->m_pPdbHeap->SetDataGuid(newGuid); #endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER } From 7a0dfcb9039f77dd0b56971918979c3da00c3ca1 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Apr 2023 15:17:24 -0700 Subject: [PATCH 10/69] Minor whitespace cleanup --- src/coreclr/inc/metadata.h | 1 + src/coreclr/md/compiler/helper.cpp | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/inc/metadata.h b/src/coreclr/inc/metadata.h index ad0adc027b066..766893bea17b8 100644 --- a/src/coreclr/inc/metadata.h +++ b/src/coreclr/inc/metadata.h @@ -1117,6 +1117,7 @@ DECLARE_INTERFACE_(IMDInternalEmit, IUnknown) STDMETHOD(SetMDUpdateMode)( ULONG updateMode, ULONG *pPreviousUpdateMode) PURE; + }; // IMDInternalEmit #ifdef FEATURE_METADATA_CUSTOM_DATA_SOURCE diff --git a/src/coreclr/md/compiler/helper.cpp b/src/coreclr/md/compiler/helper.cpp index c7d0f2d305c2e..955642853d543 100644 --- a/src/coreclr/md/compiler/helper.cpp +++ b/src/coreclr/md/compiler/helper.cpp @@ -414,7 +414,6 @@ RegMeta::GetPathSeparator( return S_OK; } // RegMeta::GetPathSeparator - #endif // FEATURE_METADATA_EMIT_PORTABLE_PDB #endif //FEATURE_METADATA_EMIT && FEATURE_METADATA_INTERNAL_APIS From ef59bedb71440b2836fcef44c57f1fbff05280a7 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 4 Jun 2024 14:37:23 -0700 Subject: [PATCH 11/69] Fix build --- src/coreclr/ilasm/assembler.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/coreclr/ilasm/assembler.cpp b/src/coreclr/ilasm/assembler.cpp index cde344e431dec..f62b4c4d58f92 100644 --- a/src/coreclr/ilasm/assembler.cpp +++ b/src/coreclr/ilasm/assembler.cpp @@ -2437,13 +2437,7 @@ HRESULT Assembler::SavePdbFile() if (FAILED(hr = (m_pPortablePdbWriter == NULL ? E_FAIL : S_OK))) goto exit; if (FAILED(hr = (m_pPortablePdbWriter->GetEmitter() == NULL ? E_FAIL : S_OK))) goto exit; -<<<<<<< HEAD if (FAILED(hr = m_pPortablePdbWriter->GetEmitter()->Save(m_wzPdbFileName, NULL))) goto exit; -======= - if (FAILED(hr = m_pCeeFileGen->GetEntryPoint(m_pCeeFile, &entryPoint))) goto exit; - if (FAILED(hr = m_pPortablePdbWriter->BuildPdbStream(m_pEmitter, entryPoint))) goto exit; - if (FAILED(hr = m_pPortablePdbWriter->GetEmitter()->Save(m_wzPdbFileName, 0))) goto exit; ->>>>>>> upstream/main exit: return hr; From 028a8b69be43817f3b54387137d99633af256abb Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 4 Jun 2024 14:56:11 -0700 Subject: [PATCH 12/69] Added IILAsmPortablePdbWriter interface --- src/coreclr/ilasm/portable_pdb.cpp | 12 ++++++++++-- src/coreclr/ilasm/portable_pdb.h | 9 +++++---- src/coreclr/ilasm/writer.cpp | 2 +- src/coreclr/md/compiler/regmeta.h | 3 +++ src/coreclr/md/inc/portablepdbmdi.h | 26 +++++++++++++++++++------- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/coreclr/ilasm/portable_pdb.cpp b/src/coreclr/ilasm/portable_pdb.cpp index bd897bb7f83dc..b85b48f864e34 100644 --- a/src/coreclr/ilasm/portable_pdb.cpp +++ b/src/coreclr/ilasm/portable_pdb.cpp @@ -101,6 +101,14 @@ HRESULT PortablePdbWriter::Init(IMetaDataDispenserEx2* mdDispenser) 0, IID_IMetaDataEmit3, (IUnknown**)&m_pdbEmitter); + + if (FAILED(hr)) goto exit; + + hr = mdDispenser->DefinePortablePdbScope( + CLSID_CorMetaDataRuntime, + 0, + IID_IILAsmPortablePdbWriter, + (IUnknown**)&m_ilasmPdbWriter); exit: return hr; } @@ -155,13 +163,13 @@ HRESULT PortablePdbWriter::BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef HRESULT PortablePdbWriter::ComputeSha256PdbStreamChecksum(BYTE(&checksum)[32]) { - return m_pdbEmitter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum); + return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum); } HRESULT PortablePdbWriter::ChangePdbStreamGuid(REFGUID newGuid) { m_pdbStream.id.pdbGuid = newGuid; - return m_pdbEmitter->ChangePdbStreamGuid(newGuid); + return m_ilasmPdbWriter->ChangePdbStreamGuid(newGuid); } HRESULT PortablePdbWriter::DefineDocument(char* name, GUID* language) diff --git a/src/coreclr/ilasm/portable_pdb.h b/src/coreclr/ilasm/portable_pdb.h index 1bca066ceb104..5fd69d0fbb603 100644 --- a/src/coreclr/ilasm/portable_pdb.h +++ b/src/coreclr/ilasm/portable_pdb.h @@ -63,10 +63,11 @@ class PortablePdbWriter BOOL _DefineLocalScope(mdMethodDef methodDefToken, Scope* currScope); private: - IMetaDataEmit3* m_pdbEmitter; - PORT_PDB_STREAM m_pdbStream; - DocumentList m_documentList; - Document* m_currentDocument; + IMetaDataEmit3* m_pdbEmitter; + IILAsmPortablePdbWriter* m_ilasmPdbWriter; + PORT_PDB_STREAM m_pdbStream; + DocumentList m_documentList; + Document* m_currentDocument; }; #endif diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 07abde79cbbb1..9e8f40cebee5b 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1835,7 +1835,7 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { goto cleanup; } - memcpy(pDst, hash, min(hashLength, dstSize)); + memcpy(pDst, hash, std::min(hashLength, dstSize)); status = STATUS_SUCCESS; cleanup: diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index be607296c4b40..a99f2abacf64d 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -1133,6 +1133,9 @@ class RegMeta : char *name, // [IN] Variable name. mdLocalVariable *locVarToken); // [OUT] Token of the defined variable. +//***************************************************************************** +// IILAsmPortablePdbWriter methods +//***************************************************************************** STDMETHODIMP ComputeSha256PdbStreamChecksum( // S_OK or error. HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] BYTE (&checksum)[32]); // [OUT] 256-bit Pdb checksum diff --git a/src/coreclr/md/inc/portablepdbmdi.h b/src/coreclr/md/inc/portablepdbmdi.h index 62cac609f2c60..fffdfbe95a5ff 100644 --- a/src/coreclr/md/inc/portablepdbmdi.h +++ b/src/coreclr/md/inc/portablepdbmdi.h @@ -69,13 +69,6 @@ DECLARE_INTERFACE_(IMetaDataEmit3, IMetaDataEmit2) USHORT index, // [IN] Variable index (slot). char *name, // [IN] Variable name. mdLocalVariable *locVarToken) PURE; // [OUT] Token of the defined variable. - - STDMETHOD(ComputeSha256PdbStreamChecksum)( // S_OK or error. - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] - BYTE (&checksum)[32]) PURE; // [OUT] 256-bit Pdb checksum - - STDMETHOD(ChangePdbStreamGuid)( // S_OK or error. - REFGUID newGuid) PURE; // [IN] GUID to use as the PDB GUID }; //------------------------------------- @@ -96,6 +89,25 @@ DECLARE_INTERFACE_(IMetaDataDispenserEx2, IMetaDataDispenserEx) IUnknown * *ppIUnk) PURE; // [OUT] Return interface on success. }; +//------------------------------------- +//--- IILAsmPortablePdbWriter +//------------------------------------- +// {8b2db1f0-91f5-4c99-bb07-29c878cf352a} +EXTERN_GUID(IID_IILAsmPortablePdbWriter, 0x8b2db1f0, 0x91f5, 0x4c99, 0xbb, 0x07, 0x29, 0xc8, 0x78, 0xcf, 0x35, 0x2a); + +//--- +#undef INTERFACE +#define INTERFACE IILAsmPortablePdbWriter +DECLARE_INTERFACE_(IILAsmPortablePdbWriter, IUnknown) +{ + STDMETHOD(ComputeSha256PdbStreamChecksum)( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] + BYTE (&checksum)[32]) PURE; // [OUT] 256-bit Pdb checksum + + STDMETHOD(ChangePdbStreamGuid)( // S_OK or error. + REFGUID newGuid) PURE; // [IN] GUID to use as the PDB GUID +}; + #ifdef __cplusplus } #endif From e1a10f855d72987811699e787f934131d3d58873 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 4 Jun 2024 15:44:43 -0700 Subject: [PATCH 13/69] Checking deterministic output for ILASM for all roundtrip tests --- src/tests/Common/CLRTest.Jit.targets | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 5c2a0735a414f..2dcf319e62f77 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -290,12 +290,18 @@ set DOTNET_JitPath=%CORE_ROOT%\superpmi-shim-collector.dll $(AssemblyName).dll IL-RT/$(AssemblyName).il IL-RT/$(AssemblyName).dll - <_IlasmRoundTripScriptText> + Date: Tue, 4 Jun 2024 17:29:41 -0700 Subject: [PATCH 14/69] Fixed getting interface. Testing determinism of pdb. --- src/coreclr/ilasm/portable_pdb.cpp | 6 +---- src/coreclr/md/compiler/disp.cpp | 2 ++ src/coreclr/md/compiler/regmeta.cpp | 4 ++++ src/coreclr/md/compiler/regmeta.h | 1 + src/tests/Common/CLRTest.Jit.targets | 35 ++++++++++++++++++++++++++++ 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/coreclr/ilasm/portable_pdb.cpp b/src/coreclr/ilasm/portable_pdb.cpp index b85b48f864e34..32275c6f06304 100644 --- a/src/coreclr/ilasm/portable_pdb.cpp +++ b/src/coreclr/ilasm/portable_pdb.cpp @@ -104,11 +104,7 @@ HRESULT PortablePdbWriter::Init(IMetaDataDispenserEx2* mdDispenser) if (FAILED(hr)) goto exit; - hr = mdDispenser->DefinePortablePdbScope( - CLSID_CorMetaDataRuntime, - 0, - IID_IILAsmPortablePdbWriter, - (IUnknown**)&m_ilasmPdbWriter); + hr = m_pdbEmitter->QueryInterface(IID_IILAsmPortablePdbWriter, (void**)&m_ilasmPdbWriter); exit: return hr; } diff --git a/src/coreclr/md/compiler/disp.cpp b/src/coreclr/md/compiler/disp.cpp index 4f88a64d0a815..74faf8e63b085 100644 --- a/src/coreclr/md/compiler/disp.cpp +++ b/src/coreclr/md/compiler/disp.cpp @@ -581,6 +581,8 @@ HRESULT Disp::QueryInterface(REFIID riid, void **ppUnk) #ifdef FEATURE_METADATA_EMIT_PORTABLE_PDB else if (riid == IID_IMetaDataDispenserEx2) *ppUnk = (IMetaDataDispenserEx2 *) this; + else if (riid == IID_IILAsmPortablePdbWriter) + *ppUnk = (IILAsmPortablePdbWriter *) this; #endif #ifdef FEATURE_METADATA_CUSTOM_DATA_SOURCE else if (riid == IID_IMetaDataDispenserCustom) diff --git a/src/coreclr/md/compiler/regmeta.cpp b/src/coreclr/md/compiler/regmeta.cpp index 85130f4cd4019..4bff10b137653 100644 --- a/src/coreclr/md/compiler/regmeta.cpp +++ b/src/coreclr/md/compiler/regmeta.cpp @@ -591,6 +591,10 @@ RegMeta::QueryInterface( *ppUnk = (IMetaDataEmit3 *)this; fIsInterfaceRW = true; } + else if (riid == IID_IILAsmPortablePdbWriter) + { + *ppUnk = static_cast(this); + } #endif else if (riid == IID_IMetaDataAssemblyEmit) { diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index a99f2abacf64d..925be2775144c 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -141,6 +141,7 @@ class RegMeta : , public IMetaDataEmit2 #else , public IMetaDataEmit3 + , public IILAsmPortablePdbWriter #endif , public IMetaDataAssemblyEmit #endif diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 2dcf319e62f77..ba1ebed20afa8 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -394,6 +394,41 @@ for inputAssemblyName in glob.glob("*.dll"): else: print("ILASM determinism succeeded") + # Test portable PDB determinism + + if not is_managed_debug_assembly(inputAssemblyName): + ilasmSwitches = ilasmSwitches + " -DEBUG" + + pdbName = inputAssemblyName.replace('.dll', '.pdb') + + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' + print(ilasm_args) + proc = subprocess.Popen(ilasm_args, shell=True) + + try: + proc.communicate() + except: + proc.kill() + sys.exit(1) + + sha256_hash = sha256sum(pdbName) + + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' + print(ilasm_args) + proc = subprocess.Popen(ilasm_args, shell=True) + + try: + proc.communicate() + except: + proc.kill() + sys.exit(1) + + if sha256_hash != sha256sum(pdbName): + print("ILASM PDB determinism failed") + sys.exit(1) + else: + print("ILASM PDB determinism succeeded") + print("") print("") From c023315d29f26caf81935fa4b553255131c56b97 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 4 Jun 2024 17:31:29 -0700 Subject: [PATCH 15/69] add comment --- src/tests/Common/CLRTest.Jit.targets | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index ba1ebed20afa8..e508c41df8843 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -366,6 +366,8 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) + # Test determinism + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) proc = subprocess.Popen(ilasm_args, shell=True) @@ -394,7 +396,7 @@ for inputAssemblyName in glob.glob("*.dll"): else: print("ILASM determinism succeeded") - # Test portable PDB determinism + # Test PDB determinism if not is_managed_debug_assembly(inputAssemblyName): ilasmSwitches = ilasmSwitches + " -DEBUG" From 9e14b48681aef58cc3bd7b96a130e6e33b930c34 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 4 Jun 2024 17:56:25 -0700 Subject: [PATCH 16/69] Remove use of goto --- src/coreclr/ilasm/writer.cpp | 42 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 9e8f40cebee5b..849416553fa86 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -298,7 +298,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) param.debugDirData = data; - DebugDirectoryEntry entry = { 0 }; + DebugDirectoryEntry entry = {}; entry.debugDirIDD = debugDirIDD; entry.debugDirDataSize = param.debugDirDataSize; entry.debugDirData = param.debugDirData; @@ -343,6 +343,19 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) memcpy_s(pdbChecksumData + pdbChecksumOffset, pdbChecksumSize, &pdbChecksum, sizeof(pdbChecksum)); // Checksum /* END PDB CHECKSUM */ + auto finish = + [&](HRESULT hr) { + if (codeViewData) + { + delete [] codeViewData; + } + if (pdbChecksumData) + { + delete [] pdbChecksumData; + } + return hr; + }; + // CodeView Entry hr = addEntry( @@ -355,7 +368,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) /* data */ codeViewData ); if (FAILED(hr)) - goto Exit; + return finish(hr); // Pdb Checksum Entry hr = @@ -369,7 +382,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) /* data */ pdbChecksumData ); if (FAILED(hr)) - goto Exit; + return finish(hr); if (m_fDeterministic) { @@ -385,7 +398,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) /* data */ NULL ); if (FAILED(hr)) - goto Exit; + return finish(hr); } HCEESECTION sec = m_pILSection; @@ -408,12 +421,12 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) totalSize, 4, (void**) &de))) - goto Exit; + return finish(hr); // Where did we get that memory? if (FAILED(hr = m_pCeeFileGen->GetSectionDataLen(sec, &deOffset))) - goto Exit; + return finish(hr); deOffset -= totalSize; @@ -423,7 +436,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) IMAGE_DIRECTORY_ENTRY_DEBUG, totalEntrySize, deOffset))) - goto Exit; + return finish(hr); ULONG rawDataOffset = deOffset + totalEntrySize; @@ -449,7 +462,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) offsetof(IMAGE_DEBUG_DIRECTORY, PointerToRawData), sec, srRelocFilePos))) - goto Exit; + return finish(hr); if (FAILED(hr = m_pCeeFileGen->AddSectionReloc( sec, @@ -457,7 +470,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) offsetof(IMAGE_DEBUG_DIRECTORY, AddressOfRawData), sec, srRelocAbsolute))) - goto Exit; + return finish(hr); } // Copy the debug directory into the section. @@ -473,16 +486,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) dataOffset += entry.debugDirDataSize; } -Exit: - if (codeViewData) - { - delete [] codeViewData; - } - if (pdbChecksumData) - { - delete [] pdbChecksumData; - } - return hr; + return finish(hr); } //#ifdef EXPORT_DIR_ENABLED HRESULT Assembler::CreateExportDirectory() From 4038a1fdb2efd2d1d6a1ee1ee5157299345967f6 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 4 Jun 2024 19:10:55 -0700 Subject: [PATCH 17/69] Added sha256 for non-win32 --- src/coreclr/ilasm/writer.cpp | 163 ++++++++++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 2 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 849416553fa86..7f11377d2e70f 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -10,6 +10,157 @@ #include "ceefilegenwriter.h" +#if !defined(_WIN32) +#define SHA256_BLOCK_SIZE 32 + +typedef struct { + unsigned char data[64]; + unsigned int datalen; + unsigned long long bitlen; + unsigned int state[8]; +} SHA256_CTX; + +#include +#include + +#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) +#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) +#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) +#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) + +static const unsigned int k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) +{ + WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; + + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + for ( ; i < 64; ++i) + m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + + a = ctx->state[0]; + b = ctx->state[1]; + c = ctx->state[2]; + d = ctx->state[3]; + e = ctx->state[4]; + f = ctx->state[5]; + g = ctx->state[6]; + h = ctx->state[7]; + + for (i = 0; i < 64; ++i) { + t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i]; + t2 = EP0(a) + MAJ(a,b,c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; + ctx->state[4] += e; + ctx->state[5] += f; + ctx->state[6] += g; + ctx->state[7] += h; +} + +void sha256_init(SHA256_CTX *ctx) +{ + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; + ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; + ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; + ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; + ctx->state[7] = 0x5be0cd19; +} + +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) +{ + WORD i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void sha256_final(SHA256_CTX *ctx, BYTE hash[]) +{ + WORD i; + + i = ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[63] = ctx->bitlen; + ctx->data[62] = ctx->bitlen >> 8; + ctx->data[61] = ctx->bitlen >> 16; + ctx->data[60] = ctx->bitlen >> 24; + ctx->data[59] = ctx->bitlen >> 32; + ctx->data[58] = ctx->bitlen >> 40; + ctx->data[57] = ctx->bitlen >> 48; + ctx->data[56] = ctx->bitlen >> 56; + sha256_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and SHA uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; + } +} +#endif + #ifndef _MSC_VER //cloned definition from ntimage.h that is removed for non MSVC builds typedef VOID @@ -1795,8 +1946,17 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { + BYTE hash[32]; // 256 bits + #if !defined(_WIN32) - // TODO: Use openssl? + SHA256_CTX ctx; + + sha256_init(&ctx); + sha256_update(&ctx, pSrc, srcSize); + sha256_final(&ctx, hash); + + memcpy(pDst, hash, std::min((DWORD)SHA256_BLOCK_SIZE, dstSize)); + return S_OK; #else NTSTATUS status; @@ -1804,7 +1964,6 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) BCRYPT_ALG_HANDLE algHandle = NULL; BCRYPT_HASH_HANDLE hashHandle = NULL; - BYTE hash[32]; // 256 bits DWORD hashLength = 0; DWORD resultLength = 0; status = BCryptOpenAlgorithmProvider(&algHandle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_HASH_REUSABLE_FLAG); From 69fadf4c5b2a83b3a81fd5084e0ec4c233966fe3 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 5 Jun 2024 16:33:20 -0700 Subject: [PATCH 18/69] Using different impl for getting sha256 of a file in python. Using sha256 in c. --- src/coreclr/ilasm/writer.cpp | 114 +++++++-------------------- src/tests/Common/CLRTest.Jit.targets | 17 ++-- 2 files changed, 39 insertions(+), 92 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 7f11377d2e70f..77cf8ad4989f4 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -10,19 +10,18 @@ #include "ceefilegenwriter.h" -#if !defined(_WIN32) +#include +#include + #define SHA256_BLOCK_SIZE 32 typedef struct { - unsigned char data[64]; - unsigned int datalen; - unsigned long long bitlen; - unsigned int state[8]; + BYTE data[64]; + UINT datalen; + ULONG bitlen; + UINT state[8]; } SHA256_CTX; -#include -#include - #define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) #define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) @@ -51,20 +50,20 @@ void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) for (i = 0, j = 0; i < 16; ++i, j += 4) m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); for ( ; i < 64; ++i) - m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + m[i] = (WORD)SIG1((UINT)(m[i - 2])) + m[i - 7] + (WORD)SIG0((UINT)(m[i - 15])) + m[i - 16]; - a = ctx->state[0]; - b = ctx->state[1]; - c = ctx->state[2]; - d = ctx->state[3]; - e = ctx->state[4]; - f = ctx->state[5]; - g = ctx->state[6]; - h = ctx->state[7]; + a = (WORD)ctx->state[0]; + b = (WORD)ctx->state[1]; + c = (WORD)ctx->state[2]; + d = (WORD)ctx->state[3]; + e = (WORD)ctx->state[4]; + f = (WORD)ctx->state[5]; + g = (WORD)ctx->state[6]; + h = (WORD)ctx->state[7]; for (i = 0; i < 64; ++i) { - t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i]; - t2 = EP0(a) + MAJ(a,b,c); + t1 = (WORD)(h + (WORD)EP1((UINT)e) + CH(e,f,g) + k[i] + m[i]); + t2 = (WORD)EP0((UINT)a) + MAJ(a,b,c); h = g; g = f; f = e; @@ -118,7 +117,7 @@ void sha256_final(SHA256_CTX *ctx, BYTE hash[]) { WORD i; - i = ctx->datalen; + i = (WORD)ctx->datalen; // Pad whatever data is left in the buffer. if (ctx->datalen < 56) { @@ -136,14 +135,14 @@ void sha256_final(SHA256_CTX *ctx, BYTE hash[]) // Append to the padding the total message's length in bits and transform. ctx->bitlen += ctx->datalen * 8; - ctx->data[63] = ctx->bitlen; - ctx->data[62] = ctx->bitlen >> 8; - ctx->data[61] = ctx->bitlen >> 16; - ctx->data[60] = ctx->bitlen >> 24; - ctx->data[59] = ctx->bitlen >> 32; - ctx->data[58] = ctx->bitlen >> 40; - ctx->data[57] = ctx->bitlen >> 48; - ctx->data[56] = ctx->bitlen >> 56; + ctx->data[63] = (unsigned char)ctx->bitlen; + ctx->data[62] = (unsigned char)(ctx->bitlen >> 8); + ctx->data[61] = (unsigned char)(ctx->bitlen >> 16); + ctx->data[60] = (unsigned char)(ctx->bitlen >> 24); + ctx->data[59] = (unsigned char)(ctx->bitlen >> 32); + ctx->data[58] = (unsigned char)(ctx->bitlen >> 40); + ctx->data[57] = (unsigned char)(ctx->bitlen >> 48); + ctx->data[56] = (unsigned char)(ctx->bitlen >> 56); sha256_transform(ctx, ctx->data); // Since this implementation uses little endian byte ordering and SHA uses big endian, @@ -159,7 +158,6 @@ void sha256_final(SHA256_CTX *ctx, BYTE hash[]) hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; } } -#endif #ifndef _MSC_VER //cloned definition from ntimage.h that is removed for non MSVC builds @@ -1946,9 +1944,7 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - BYTE hash[32]; // 256 bits - -#if !defined(_WIN32) + BYTE hash[SHA256_BLOCK_SIZE]; SHA256_CTX ctx; sha256_init(&ctx); @@ -1958,60 +1954,6 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) memcpy(pDst, hash, std::min((DWORD)SHA256_BLOCK_SIZE, dstSize)); return S_OK; -#else - NTSTATUS status; - - BCRYPT_ALG_HANDLE algHandle = NULL; - BCRYPT_HASH_HANDLE hashHandle = NULL; - - DWORD hashLength = 0; - DWORD resultLength = 0; - status = BCryptOpenAlgorithmProvider(&algHandle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_HASH_REUSABLE_FLAG); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - status = BCryptGetProperty(algHandle, BCRYPT_HASH_LENGTH, (PBYTE)&hashLength, sizeof(hashLength), &resultLength, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - if (hashLength != 32) - { - status = STATUS_NO_MEMORY; - goto cleanup; - } - status = BCryptCreateHash(algHandle, &hashHandle, NULL, 0, NULL, 0, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - - status = BCryptHashData(hashHandle, pSrc, srcSize, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - - status = BCryptFinishHash(hashHandle, hash, hashLength, 0); - if(!NT_SUCCESS(status)) - { - goto cleanup; - } - memcpy(pDst, hash, std::min(hashLength, dstSize)); - status = STATUS_SUCCESS; - -cleanup: - if (NULL != hashHandle) - { - BCryptDestroyHash(hashHandle); - } - if(NULL != algHandle) - { - BCryptCloseAlgorithmProvider(algHandle, 0); - } - return status; -#endif } #ifdef _PREFAST_ diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index e508c41df8843..4b75fbfa23dd3 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -299,9 +299,14 @@ import sys import glob import hashlib -def sha256sum(filename): +def hash_file(filename): + h = hashlib.sha256() + b = bytearray(128*1024) + mv = memoryview(b) with open(filename, 'rb', buffering=0) as f: - return hashlib.file_digest(f, 'sha256').hexdigest() + while n := f.readinto(mv): + h.update(mv[:n]) + return h.hexdigest() def is_managed_assembly(file): proc = subprocess.Popen([f'{os.environ["CORE_ROOT"]}/corerun', f'{os.environ["CORE_ROOT"]}/AssemblyChecker/AssemblyChecker.dll', file]) @@ -378,7 +383,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - sha256_hash = sha256sum(inputAssemblyName) + hash = hash_file(inputAssemblyName) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) @@ -390,7 +395,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - if sha256_hash != sha256sum(inputAssemblyName): + if hash != hash_file(inputAssemblyName): print("ILASM determinism failed") sys.exit(1) else: @@ -413,7 +418,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - sha256_hash = sha256sum(pdbName) + hash = hash_file(pdbName) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) @@ -425,7 +430,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - if sha256_hash != sha256sum(pdbName): + if hash != hash_file(pdbName): print("ILASM PDB determinism failed") sys.exit(1) else: From 936f0a3916f47dd87c435d170b53a7b396d1e417 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 5 Jun 2024 16:53:18 -0700 Subject: [PATCH 19/69] Trying to fix build --- src/coreclr/ilasm/writer.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 77cf8ad4989f4..67d045158b060 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -18,7 +18,7 @@ typedef struct { BYTE data[64]; UINT datalen; - ULONG bitlen; + unsigned long long bitlen; UINT state[8]; } SHA256_CTX; @@ -135,14 +135,14 @@ void sha256_final(SHA256_CTX *ctx, BYTE hash[]) // Append to the padding the total message's length in bits and transform. ctx->bitlen += ctx->datalen * 8; - ctx->data[63] = (unsigned char)ctx->bitlen; - ctx->data[62] = (unsigned char)(ctx->bitlen >> 8); - ctx->data[61] = (unsigned char)(ctx->bitlen >> 16); - ctx->data[60] = (unsigned char)(ctx->bitlen >> 24); - ctx->data[59] = (unsigned char)(ctx->bitlen >> 32); - ctx->data[58] = (unsigned char)(ctx->bitlen >> 40); - ctx->data[57] = (unsigned char)(ctx->bitlen >> 48); - ctx->data[56] = (unsigned char)(ctx->bitlen >> 56); + ctx->data[63] = (BYTE)ctx->bitlen; + ctx->data[62] = (BYTE)(ctx->bitlen >> 8); + ctx->data[61] = (BYTE)(ctx->bitlen >> 16); + ctx->data[60] = (BYTE)(ctx->bitlen >> 24); + ctx->data[59] = (BYTE)(ctx->bitlen >> 32); + ctx->data[58] = (BYTE)(ctx->bitlen >> 40); + ctx->data[57] = (BYTE)(ctx->bitlen >> 48); + ctx->data[56] = (BYTE)(ctx->bitlen >> 56); sha256_transform(ctx, ctx->data); // Since this implementation uses little endian byte ordering and SHA uses big endian, From d6b42582ec28a7183260e797b95d574611874e80 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 5 Jun 2024 19:48:27 -0700 Subject: [PATCH 20/69] Fix build --- src/tests/Common/CLRTest.Jit.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 4b75fbfa23dd3..cf528d5e959e4 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -304,7 +304,7 @@ def hash_file(filename): b = bytearray(128*1024) mv = memoryview(b) with open(filename, 'rb', buffering=0) as f: - while n := f.readinto(mv): + for n in iter(lambda : f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest() From 9c1a645553f130c48e5d90cbcffb62e463d32fc6 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Thu, 6 Jun 2024 10:08:40 -0700 Subject: [PATCH 21/69] disable testing determinism due to timeouts --- src/tests/Common/CLRTest.Jit.targets | 85 ++++++++++++++-------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index cf528d5e959e4..3c2bf30bf913f 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -371,8 +371,6 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - # Test determinism - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) proc = subprocess.Popen(ilasm_args, shell=True) @@ -383,58 +381,63 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - hash = hash_file(inputAssemblyName) + test_det = False - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' - print(ilasm_args) - proc = subprocess.Popen(ilasm_args, shell=True) + # Test determinism + if test_det: - try: - proc.communicate() - except: - proc.kill() + hash = hash_file(inputAssemblyName) + + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' + print(ilasm_args) + proc = subprocess.Popen(ilasm_args, shell=True) + + try: + proc.communicate() + except: + proc.kill() + sys.exit(1) + + if hash != hash_file(inputAssemblyName): + print("ILASM determinism failed") sys.exit(1) + else: + print("ILASM determinism succeeded") - if hash != hash_file(inputAssemblyName): - print("ILASM determinism failed") - sys.exit(1) - else: - print("ILASM determinism succeeded") + # Test PDB determinism - # Test PDB determinism + if not is_managed_debug_assembly(inputAssemblyName): + ilasmSwitches = ilasmSwitches + " -DEBUG" - if not is_managed_debug_assembly(inputAssemblyName): - ilasmSwitches = ilasmSwitches + " -DEBUG" + pdbName = inputAssemblyName.replace('.dll', '.pdb') - pdbName = inputAssemblyName.replace('.dll', '.pdb') + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' + print(ilasm_args) + proc = subprocess.Popen(ilasm_args, shell=True) - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' - print(ilasm_args) - proc = subprocess.Popen(ilasm_args, shell=True) + try: + proc.communicate() + except: + proc.kill() + sys.exit(1) - try: - proc.communicate() - except: - proc.kill() - sys.exit(1) + hash = hash_file(pdbName) - hash = hash_file(pdbName) + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' + print(ilasm_args) + proc = subprocess.Popen(ilasm_args, shell=True) - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' - print(ilasm_args) - proc = subprocess.Popen(ilasm_args, shell=True) + try: + proc.communicate() + except: + proc.kill() + sys.exit(1) - try: - proc.communicate() - except: - proc.kill() + if hash != hash_file(pdbName): + print("ILASM PDB determinism failed") sys.exit(1) - - if hash != hash_file(pdbName): - print("ILASM PDB determinism failed") - sys.exit(1) - else: - print("ILASM PDB determinism succeeded") + else: + print("ILASM PDB determinism succeeded") print("") From 3331d9d4b04b2b0af76c484599695328c6fdf46a Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 23 Jul 2024 13:33:48 -0700 Subject: [PATCH 22/69] Temporarily disable emitting debug directories to see if the tests time out in CI --- src/coreclr/ilasm/writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index d8545365a3dde..523354087cdbe 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1619,7 +1619,7 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(pdbGuid))) goto exit; } - if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; + //if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; } if (FAILED(hr=m_pCeeFileGen->SetOutputFileName(m_pCeeFile, pwzOutputFilename))) goto exit; From 8e357b8e8bda15cada300a1cf5549fade75a2153 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 24 Jul 2024 11:24:36 -0700 Subject: [PATCH 23/69] Renable deterministic tests and debug directory creation --- src/coreclr/ilasm/writer.cpp | 2 +- src/tests/Common/CLRTest.Jit.targets | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 523354087cdbe..d8545365a3dde 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1619,7 +1619,7 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(pdbGuid))) goto exit; } - //if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; + if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; } if (FAILED(hr=m_pCeeFileGen->SetOutputFileName(m_pCeeFile, pwzOutputFilename))) goto exit; diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 3c2bf30bf913f..b0fa17b34b1dc 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -381,7 +381,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - test_det = False + test_det = True # Test determinism if test_det: From 8867295a9fca57bafe2e1838ee914cb146501d03 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 24 Jul 2024 11:56:14 -0700 Subject: [PATCH 24/69] Added sha256.cpp --- src/coreclr/ilasm/assembler.h | 2 - src/coreclr/ilasm/portable_pdb.cpp | 2 +- src/coreclr/ilasm/writer.cpp | 164 +------------------------ src/coreclr/md/compiler/emit.cpp | 3 +- src/coreclr/md/compiler/regmeta.h | 1 - src/coreclr/md/enc/CMakeLists.txt | 2 + src/coreclr/md/enc/pdbheap.cpp | 5 +- src/coreclr/md/enc/sha256.cpp | 183 ++++++++++++++++++++++++++++ src/coreclr/md/inc/pdbheap.h | 2 +- src/coreclr/md/inc/portablepdbmdi.h | 1 - src/coreclr/md/inc/sha256.h | 16 +++ 11 files changed, 208 insertions(+), 173 deletions(-) create mode 100644 src/coreclr/md/enc/sha256.cpp create mode 100644 src/coreclr/md/inc/sha256.h diff --git a/src/coreclr/ilasm/assembler.h b/src/coreclr/ilasm/assembler.h index ab6ced59262cc..59d0580f623fe 100644 --- a/src/coreclr/ilasm/assembler.h +++ b/src/coreclr/ilasm/assembler.h @@ -1258,8 +1258,6 @@ class Assembler { }; -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize); - #endif // Assember_h #ifdef _MSC_VER diff --git a/src/coreclr/ilasm/portable_pdb.cpp b/src/coreclr/ilasm/portable_pdb.cpp index 32275c6f06304..7147c054714df 100644 --- a/src/coreclr/ilasm/portable_pdb.cpp +++ b/src/coreclr/ilasm/portable_pdb.cpp @@ -159,7 +159,7 @@ HRESULT PortablePdbWriter::BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef HRESULT PortablePdbWriter::ComputeSha256PdbStreamChecksum(BYTE(&checksum)[32]) { - return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum); + return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(checksum); } HRESULT PortablePdbWriter::ChangePdbStreamGuid(REFGUID newGuid) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index d8545365a3dde..f40069f16b0aa 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -9,155 +9,7 @@ #include "assembler.h" #include "ceefilegenwriter.h" - -#include -#include - -#define SHA256_BLOCK_SIZE 32 - -typedef struct { - BYTE data[64]; - UINT datalen; - unsigned long long bitlen; - UINT state[8]; -} SHA256_CTX; - -#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) -#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) - -#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) -#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) -#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) -#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) -#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) -#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) - -static const unsigned int k[64] = { - 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, - 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, - 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, - 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, - 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, - 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, - 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, - 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 -}; - -void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) -{ - WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; - - for (i = 0, j = 0; i < 16; ++i, j += 4) - m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); - for ( ; i < 64; ++i) - m[i] = (WORD)SIG1((UINT)(m[i - 2])) + m[i - 7] + (WORD)SIG0((UINT)(m[i - 15])) + m[i - 16]; - - a = (WORD)ctx->state[0]; - b = (WORD)ctx->state[1]; - c = (WORD)ctx->state[2]; - d = (WORD)ctx->state[3]; - e = (WORD)ctx->state[4]; - f = (WORD)ctx->state[5]; - g = (WORD)ctx->state[6]; - h = (WORD)ctx->state[7]; - - for (i = 0; i < 64; ++i) { - t1 = (WORD)(h + (WORD)EP1((UINT)e) + CH(e,f,g) + k[i] + m[i]); - t2 = (WORD)EP0((UINT)a) + MAJ(a,b,c); - h = g; - g = f; - f = e; - e = d + t1; - d = c; - c = b; - b = a; - a = t1 + t2; - } - - ctx->state[0] += a; - ctx->state[1] += b; - ctx->state[2] += c; - ctx->state[3] += d; - ctx->state[4] += e; - ctx->state[5] += f; - ctx->state[6] += g; - ctx->state[7] += h; -} - -void sha256_init(SHA256_CTX *ctx) -{ - ctx->datalen = 0; - ctx->bitlen = 0; - ctx->state[0] = 0x6a09e667; - ctx->state[1] = 0xbb67ae85; - ctx->state[2] = 0x3c6ef372; - ctx->state[3] = 0xa54ff53a; - ctx->state[4] = 0x510e527f; - ctx->state[5] = 0x9b05688c; - ctx->state[6] = 0x1f83d9ab; - ctx->state[7] = 0x5be0cd19; -} - -void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) -{ - WORD i; - - for (i = 0; i < len; ++i) { - ctx->data[ctx->datalen] = data[i]; - ctx->datalen++; - if (ctx->datalen == 64) { - sha256_transform(ctx, ctx->data); - ctx->bitlen += 512; - ctx->datalen = 0; - } - } -} - -void sha256_final(SHA256_CTX *ctx, BYTE hash[]) -{ - WORD i; - - i = (WORD)ctx->datalen; - - // Pad whatever data is left in the buffer. - if (ctx->datalen < 56) { - ctx->data[i++] = 0x80; - while (i < 56) - ctx->data[i++] = 0x00; - } - else { - ctx->data[i++] = 0x80; - while (i < 64) - ctx->data[i++] = 0x00; - sha256_transform(ctx, ctx->data); - memset(ctx->data, 0, 56); - } - - // Append to the padding the total message's length in bits and transform. - ctx->bitlen += ctx->datalen * 8; - ctx->data[63] = (BYTE)ctx->bitlen; - ctx->data[62] = (BYTE)(ctx->bitlen >> 8); - ctx->data[61] = (BYTE)(ctx->bitlen >> 16); - ctx->data[60] = (BYTE)(ctx->bitlen >> 24); - ctx->data[59] = (BYTE)(ctx->bitlen >> 32); - ctx->data[58] = (BYTE)(ctx->bitlen >> 40); - ctx->data[57] = (BYTE)(ctx->bitlen >> 48); - ctx->data[56] = (BYTE)(ctx->bitlen >> 56); - sha256_transform(ctx, ctx->data); - - // Since this implementation uses little endian byte ordering and SHA uses big endian, - // reverse all the bytes when copying the final state to the output hash. - for (i = 0; i < 4; ++i) { - hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; - hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; - hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; - hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; - hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; - hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; - hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; - hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; - } -} +#include "sha256.h" #ifndef _MSC_VER //cloned definition from ntimage.h that is removed for non MSVC builds @@ -1942,20 +1794,6 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) return hr; } -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) -{ - BYTE hash[SHA256_BLOCK_SIZE]; - SHA256_CTX ctx; - - sha256_init(&ctx); - sha256_update(&ctx, pSrc, srcSize); - sha256_final(&ctx, hash); - - memcpy(pDst, hash, std::min((DWORD)SHA256_BLOCK_SIZE, dstSize)); - - return S_OK; -} - #ifdef _PREFAST_ #pragma warning(pop) #endif diff --git a/src/coreclr/md/compiler/emit.cpp b/src/coreclr/md/compiler/emit.cpp index 3c6dae259e474..600e4f11898de 100644 --- a/src/coreclr/md/compiler/emit.cpp +++ b/src/coreclr/md/compiler/emit.cpp @@ -2119,13 +2119,12 @@ STDMETHODIMP RegMeta::DefineLocalVariable( // S_OK or error. // ComputeSha256PdbStreamChecksum //******************************************************************************* STDMETHODIMP RegMeta::ComputeSha256PdbStreamChecksum( - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]) { #ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER return E_NOTIMPL; #else //!FEATURE_METADATA_EMIT_IN_DEBUGGER - return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); + return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(checksum); #endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER } diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index 925be2775144c..bcbc41fc479c1 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -1138,7 +1138,6 @@ class RegMeta : // IILAsmPortablePdbWriter methods //***************************************************************************** STDMETHODIMP ComputeSha256PdbStreamChecksum( // S_OK or error. - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] BYTE (&checksum)[32]); // [OUT] 256-bit Pdb checksum STDMETHODIMP ChangePdbStreamGuid( // S_OK or error. diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index 5b96ed8d87c97..cc9aa83d57825 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -4,6 +4,7 @@ set(MDRUNTIMERW_SOURCES metamodelrw.cpp peparse.cpp rwutil.cpp + sha256.cpp stgio.cpp stgtiggerstorage.cpp stgtiggerstream.cpp @@ -33,6 +34,7 @@ set(MDRUNTIMERW_HEADERS ../inc/portablepdbmdds.h ../inc/portablepdbmdi.h ../inc/rwutil.h + ../inc/sha256.h ../inc/stgio.h ../inc/stgtiggerstorage.h ../inc/stgtiggerstream.h diff --git a/src/coreclr/md/enc/pdbheap.cpp b/src/coreclr/md/enc/pdbheap.cpp index d402cddc0a731..a575354b9d541 100644 --- a/src/coreclr/md/enc/pdbheap.cpp +++ b/src/coreclr/md/enc/pdbheap.cpp @@ -3,6 +3,7 @@ #include "stdafx.h" #include "pdbheap.h" +#include "sha256.h" PdbHeap::PdbHeap() : m_data(NULL), m_size(0) { @@ -79,10 +80,10 @@ HRESULT PdbHeap::SetDataGuid(REFGUID newGuid) } __checkReturn -HRESULT PdbHeap::ComputeSha256Checksum(HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]) +HRESULT PdbHeap::ComputeSha256Checksum(BYTE (&checksum)[32]) { _ASSERTE(m_size >= sizeof(PDB_ID)); - return computeSha256(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); + return Sha256Hash(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); } __checkReturn diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp new file mode 100644 index 0000000000000..df14577630006 --- /dev/null +++ b/src/coreclr/md/enc/sha256.cpp @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +//***************************************************************************** +// sha256.cpp +// + +// +// contains implementation of sha256 hash algorithm +// +//***************************************************************************** + +//***************************************************************************** +// SHA256 IMPL: https://github.com/solokeys/solo1/blob/master/crypto/sha256/sha256.c +// * Author: Brad Conte (brad AT bradconte.com) +// * Copyright: +// * Disclaimer: This code is presented "as is" without any guarantees. +// * Details: Implementation of the SHA-256 hashing algorithm. +// SHA-256 is one of the three algorithms in the SHA2 +// specification. The others, SHA-384 and SHA-512, are not +// offered in this implementation. +// Algorithm specification can be found here: +// * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf +// This implementation uses little endian byte order. +//***************************************************************************** +#define SHA256_BLOCK_SIZE 32 + +typedef struct { + BYTE data[64]; + UINT datalen; + unsigned long long bitlen; + UINT state[8]; +} SHA256_CTX; + +#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) +#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) +#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) +#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) + +static const unsigned int k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) +{ + WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; + + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + for ( ; i < 64; ++i) + m[i] = (WORD)SIG1((UINT)(m[i - 2])) + m[i - 7] + (WORD)SIG0((UINT)(m[i - 15])) + m[i - 16]; + + a = (WORD)ctx->state[0]; + b = (WORD)ctx->state[1]; + c = (WORD)ctx->state[2]; + d = (WORD)ctx->state[3]; + e = (WORD)ctx->state[4]; + f = (WORD)ctx->state[5]; + g = (WORD)ctx->state[6]; + h = (WORD)ctx->state[7]; + + for (i = 0; i < 64; ++i) { + t1 = (WORD)(h + (WORD)EP1((UINT)e) + CH(e,f,g) + k[i] + m[i]); + t2 = (WORD)EP0((UINT)a) + MAJ(a,b,c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; + ctx->state[4] += e; + ctx->state[5] += f; + ctx->state[6] += g; + ctx->state[7] += h; +} + +void sha256_init(SHA256_CTX *ctx) +{ + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; + ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; + ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; + ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; + ctx->state[7] = 0x5be0cd19; +} + +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) +{ + WORD i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void sha256_final(SHA256_CTX *ctx, BYTE hash[]) +{ + WORD i; + + i = (WORD)ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[63] = (BYTE)ctx->bitlen; + ctx->data[62] = (BYTE)(ctx->bitlen >> 8); + ctx->data[61] = (BYTE)(ctx->bitlen >> 16); + ctx->data[60] = (BYTE)(ctx->bitlen >> 24); + ctx->data[59] = (BYTE)(ctx->bitlen >> 32); + ctx->data[58] = (BYTE)(ctx->bitlen >> 40); + ctx->data[57] = (BYTE)(ctx->bitlen >> 48); + ctx->data[56] = (BYTE)(ctx->bitlen >> 56); + sha256_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and SHA uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; + } +} + +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +{ + BYTE hash[SHA256_BLOCK_SIZE]; + SHA256_CTX ctx; + + sha256_init(&ctx); + sha256_update(&ctx, pSrc, srcSize); + sha256_final(&ctx, hash); + + memcpy(pDst, hash, std::min((DWORD)SHA256_BLOCK_SIZE, dstSize)); + + return S_OK; +} \ No newline at end of file diff --git a/src/coreclr/md/inc/pdbheap.h b/src/coreclr/md/inc/pdbheap.h index 88bae893f86d7..117a07fe0b354 100644 --- a/src/coreclr/md/inc/pdbheap.h +++ b/src/coreclr/md/inc/pdbheap.h @@ -22,7 +22,7 @@ class PdbHeap __checkReturn HRESULT SetData(PORT_PDB_STREAM* data); __checkReturn HRESULT SetDataGuid(REFGUID newGuid); - __checkReturn HRESULT ComputeSha256Checksum(HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]); + __checkReturn HRESULT ComputeSha256Checksum(BYTE (&checksum)[32]); __checkReturn HRESULT SaveToStream(IStream* stream); BOOL IsEmpty(); ULONG GetSize(); diff --git a/src/coreclr/md/inc/portablepdbmdi.h b/src/coreclr/md/inc/portablepdbmdi.h index fffdfbe95a5ff..a05bc163c2876 100644 --- a/src/coreclr/md/inc/portablepdbmdi.h +++ b/src/coreclr/md/inc/portablepdbmdi.h @@ -101,7 +101,6 @@ EXTERN_GUID(IID_IILAsmPortablePdbWriter, 0x8b2db1f0, 0x91f5, 0x4c99, 0xbb, 0x07, DECLARE_INTERFACE_(IILAsmPortablePdbWriter, IUnknown) { STDMETHOD(ComputeSha256PdbStreamChecksum)( // S_OK or error. - HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] BYTE (&checksum)[32]) PURE; // [OUT] 256-bit Pdb checksum STDMETHOD(ChangePdbStreamGuid)( // S_OK or error. diff --git a/src/coreclr/md/inc/sha256.h b/src/coreclr/md/inc/sha256.h new file mode 100644 index 0000000000000..6374dde7316a8 --- /dev/null +++ b/src/coreclr/md/inc/sha256.h @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +//***************************************************************************** +// sha256.h +// + +// +// contains implementation of sha256 hash algorithm +// +//***************************************************************************** +#ifndef __sha256__h__ +#define __sha256__h__ + +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize); + +#endif // __sha256__h__ From 6dd077fab3cc60d31854f3e210909f58360ca737 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 24 Jul 2024 12:59:16 -0700 Subject: [PATCH 25/69] Remove sha256 impl for now --- src/coreclr/md/enc/sha256.cpp | 171 +--------------------------------- 1 file changed, 2 insertions(+), 169 deletions(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index df14577630006..eaf75aaa42096 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -9,175 +9,8 @@ // //***************************************************************************** -//***************************************************************************** -// SHA256 IMPL: https://github.com/solokeys/solo1/blob/master/crypto/sha256/sha256.c -// * Author: Brad Conte (brad AT bradconte.com) -// * Copyright: -// * Disclaimer: This code is presented "as is" without any guarantees. -// * Details: Implementation of the SHA-256 hashing algorithm. -// SHA-256 is one of the three algorithms in the SHA2 -// specification. The others, SHA-384 and SHA-512, are not -// offered in this implementation. -// Algorithm specification can be found here: -// * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf -// This implementation uses little endian byte order. -//***************************************************************************** -#define SHA256_BLOCK_SIZE 32 - -typedef struct { - BYTE data[64]; - UINT datalen; - unsigned long long bitlen; - UINT state[8]; -} SHA256_CTX; - -#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) -#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) - -#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) -#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) -#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) -#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) -#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) -#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) - -static const unsigned int k[64] = { - 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, - 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, - 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, - 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, - 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, - 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, - 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, - 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 -}; - -void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) -{ - WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; - - for (i = 0, j = 0; i < 16; ++i, j += 4) - m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); - for ( ; i < 64; ++i) - m[i] = (WORD)SIG1((UINT)(m[i - 2])) + m[i - 7] + (WORD)SIG0((UINT)(m[i - 15])) + m[i - 16]; - - a = (WORD)ctx->state[0]; - b = (WORD)ctx->state[1]; - c = (WORD)ctx->state[2]; - d = (WORD)ctx->state[3]; - e = (WORD)ctx->state[4]; - f = (WORD)ctx->state[5]; - g = (WORD)ctx->state[6]; - h = (WORD)ctx->state[7]; - - for (i = 0; i < 64; ++i) { - t1 = (WORD)(h + (WORD)EP1((UINT)e) + CH(e,f,g) + k[i] + m[i]); - t2 = (WORD)EP0((UINT)a) + MAJ(a,b,c); - h = g; - g = f; - f = e; - e = d + t1; - d = c; - c = b; - b = a; - a = t1 + t2; - } - - ctx->state[0] += a; - ctx->state[1] += b; - ctx->state[2] += c; - ctx->state[3] += d; - ctx->state[4] += e; - ctx->state[5] += f; - ctx->state[6] += g; - ctx->state[7] += h; -} - -void sha256_init(SHA256_CTX *ctx) -{ - ctx->datalen = 0; - ctx->bitlen = 0; - ctx->state[0] = 0x6a09e667; - ctx->state[1] = 0xbb67ae85; - ctx->state[2] = 0x3c6ef372; - ctx->state[3] = 0xa54ff53a; - ctx->state[4] = 0x510e527f; - ctx->state[5] = 0x9b05688c; - ctx->state[6] = 0x1f83d9ab; - ctx->state[7] = 0x5be0cd19; -} - -void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) -{ - WORD i; - - for (i = 0; i < len; ++i) { - ctx->data[ctx->datalen] = data[i]; - ctx->datalen++; - if (ctx->datalen == 64) { - sha256_transform(ctx, ctx->data); - ctx->bitlen += 512; - ctx->datalen = 0; - } - } -} - -void sha256_final(SHA256_CTX *ctx, BYTE hash[]) -{ - WORD i; - - i = (WORD)ctx->datalen; - - // Pad whatever data is left in the buffer. - if (ctx->datalen < 56) { - ctx->data[i++] = 0x80; - while (i < 56) - ctx->data[i++] = 0x00; - } - else { - ctx->data[i++] = 0x80; - while (i < 64) - ctx->data[i++] = 0x00; - sha256_transform(ctx, ctx->data); - memset(ctx->data, 0, 56); - } - - // Append to the padding the total message's length in bits and transform. - ctx->bitlen += ctx->datalen * 8; - ctx->data[63] = (BYTE)ctx->bitlen; - ctx->data[62] = (BYTE)(ctx->bitlen >> 8); - ctx->data[61] = (BYTE)(ctx->bitlen >> 16); - ctx->data[60] = (BYTE)(ctx->bitlen >> 24); - ctx->data[59] = (BYTE)(ctx->bitlen >> 32); - ctx->data[58] = (BYTE)(ctx->bitlen >> 40); - ctx->data[57] = (BYTE)(ctx->bitlen >> 48); - ctx->data[56] = (BYTE)(ctx->bitlen >> 56); - sha256_transform(ctx, ctx->data); - - // Since this implementation uses little endian byte ordering and SHA uses big endian, - // reverse all the bytes when copying the final state to the output hash. - for (i = 0; i < 4; ++i) { - hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; - hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; - hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; - hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; - hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; - hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; - hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; - hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; - } -} - HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - BYTE hash[SHA256_BLOCK_SIZE]; - SHA256_CTX ctx; - - sha256_init(&ctx); - sha256_update(&ctx, pSrc, srcSize); - sha256_final(&ctx, hash); - - memcpy(pDst, hash, std::min((DWORD)SHA256_BLOCK_SIZE, dstSize)); - + // TODO: return S_OK; -} \ No newline at end of file +} From ce016faa4ffbd72a537e05076820eeb309de893b Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 24 Jul 2024 14:04:16 -0700 Subject: [PATCH 26/69] memset for now --- src/coreclr/md/enc/sha256.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index eaf75aaa42096..eb024efe5fa79 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -11,6 +11,6 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - // TODO: + memset(pDst, 0, dstSize); return S_OK; } From 0086e7d7a5311694e60024241ea82b17ae6ae641 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 24 Jul 2024 18:22:22 -0700 Subject: [PATCH 27/69] Temp disable det testing --- src/tests/Common/CLRTest.Jit.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index b0fa17b34b1dc..3c2bf30bf913f 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -381,7 +381,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - test_det = True + test_det = False # Test determinism if test_det: From 246e42e2b4edd63db575ccd0dd0f348d36702060 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Mon, 21 Oct 2024 18:35:19 -0400 Subject: [PATCH 28/69] Wip: windows/macos sha256 impl --- src/coreclr/md/enc/sha256.cpp | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index eb024efe5fa79..f3a8acc43b225 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -9,8 +9,93 @@ // //***************************************************************************** +#ifdef _WIN32 +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +{ + NTSTATUS status; + + BCRYPT_ALG_HANDLE algHandle = NULL; + BCRYPT_HASH_HANDLE hashHandle = NULL; + + BYTE hash[32]; // 256 bits + DWORD hashLength = 0; + DWORD resultLength = 0; + status = BCryptOpenAlgorithmProvider(&algHandle, BCRYPT_SHA256_ALGORITHM, NULL, BCRYPT_HASH_REUSABLE_FLAG); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + status = BCryptGetProperty(algHandle, BCRYPT_HASH_LENGTH, (PBYTE)&hashLength, sizeof(hashLength), &resultLength, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + if (hashLength != 32) + { + status = STATUS_NO_MEMORY; + goto cleanup; + } + status = BCryptCreateHash(algHandle, &hashHandle, NULL, 0, NULL, 0, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + status = BCryptHashData(hashHandle, pSrc, srcSize, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + status = BCryptFinishHash(hashHandle, hash, hashLength, 0); + if(!NT_SUCCESS(status)) + { + goto cleanup; + } + + memcpy(pDst, hash, min(hashLength, dstSize)); + status = S_OK; + +cleanup: + if (NULL != hashHandle) + { + BCryptDestroyHash(hashHandle); + } + if(NULL != algHandle) + { + BCryptCloseAlgorithmProvider(algHandle, 0); + } + return status; +} +#elif defined(__APPLE__) +#include +#include + +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +{ + if (dstSize < CC_SHA256_DIGEST_LENGTH) + { + return E_FAIL; + } + + // Apple's documentation states these functions return 1 on success. + CC_SHA256_CTX ctx = {{ 0 }}; + int ret = CC_SHA256_Init(&ctx); + assert(ret == 1); + + ret = CC_SHA256_Update(&ctx, pSrc, srcSize); + assert(ret == 1); + + ret = CC_SHA256_Final(pDst, &ctx); + assert(ret == 1); + + return S_OK; +} +#else +// Unsupported platform HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { memset(pDst, 0, dstSize); return S_OK; } +#endif From b65f0498e862cf6edeab4c43d7e2a3247aa25ccf Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Mon, 21 Oct 2024 19:36:12 -0400 Subject: [PATCH 29/69] Fix build --- src/coreclr/ilasm/assembler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/ilasm/assembler.cpp b/src/coreclr/ilasm/assembler.cpp index c24facdfc544c..633daec451087 100644 --- a/src/coreclr/ilasm/assembler.cpp +++ b/src/coreclr/ilasm/assembler.cpp @@ -2437,7 +2437,7 @@ HRESULT Assembler::SavePdbFile() if (FAILED(hr = (m_pPortablePdbWriter == NULL ? E_FAIL : S_OK))) goto exit; if (FAILED(hr = (m_pPortablePdbWriter->GetEmitter() == NULL ? E_FAIL : S_OK))) goto exit; - if (FAILED(hr = m_pPortablePdbWriter->GetEmitter()->Save(m_wzPdbFileName, NULL))) goto exit; + if (FAILED(hr = m_pPortablePdbWriter->GetEmitter()->Save(m_wzPdbFileName, 0))) goto exit; exit: return hr; From 7cad7db58a5d91fab9dccc691d0c75bfa2233783 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 22 Oct 2024 11:28:51 -0400 Subject: [PATCH 30/69] Non-zero default timestamp --- src/coreclr/ilasm/writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index f40069f16b0aa..b90c35fecd931 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -88,7 +88,7 @@ HRESULT Assembler::InitMetaData() { // Default values for determinism. m_pPortablePdbWriter->SetGuid(GUID()); - m_pPortablePdbWriter->SetTimestamp(0); + m_pPortablePdbWriter->SetTimestamp(1); } } From d673645d6b32612c84f8ff8c477f6fdc0cd3ea6c Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 15:21:07 +0000 Subject: [PATCH 31/69] OpenSSL shim dependency --- src/coreclr/ilasm/CMakeLists.txt | 15 +++++++++++++++ src/coreclr/ilasm/main.cpp | 14 ++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/coreclr/ilasm/CMakeLists.txt b/src/coreclr/ilasm/CMakeLists.txt index cca2c6da1858e..5321371b2af64 100644 --- a/src/coreclr/ilasm/CMakeLists.txt +++ b/src/coreclr/ilasm/CMakeLists.txt @@ -10,6 +10,15 @@ add_definitions(-DFEATURE_CORECLR) include_directories(.) +if(CLR_CMAKE_TARGET_LINUX) + configure_file( + ../../native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in + ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h) + include_directories(../../native/libs/Common) + include_directories(../../native/libs/System.Security.Cryptography.Native) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) +endif(CLR_CMAKE_TARGET_LINUX) + set(ILASM_SOURCES assem.cpp writer.cpp @@ -108,6 +117,12 @@ else() ) endif(CLR_CMAKE_TARGET_WIN32) +if(CLR_CMAKE_TARGET_LINUX) + list(APPEND ILASM_LINK_LIBRARIES + System.Security.Cryptography.Native.OpenSsl-Static + ) +endif(CLR_CMAKE_TARGET_LINUX) + if(CLR_CMAKE_HOST_UNIX) target_link_libraries(ilasm diff --git a/src/coreclr/ilasm/main.cpp b/src/coreclr/ilasm/main.cpp index b9ccc45bee21d..28bede0b3975d 100644 --- a/src/coreclr/ilasm/main.cpp +++ b/src/coreclr/ilasm/main.cpp @@ -5,6 +5,12 @@ // +#ifdef __linux__ +extern "C" { + #include "openssl.h" +} +#endif + #include "ilasmpch.h" #include "asmparse.h" @@ -180,6 +186,7 @@ extern "C" int _cdecl wmain(int argc, _In_ WCHAR **argv) printf("\n/DEBUG Disable JIT optimization, create PDB file, use sequence points from PDB"); printf("\n/DEBUG=IMPL Disable JIT optimization, create PDB file, use implicit sequence points"); printf("\n/DEBUG=OPT Enable JIT optimization, create PDB file, use implicit sequence points"); + printf("\n/DET Produce deterministic outputs"); printf("\n/OPTIMIZE Optimize long instructions to short"); printf("\n/FOLD Fold the identical method bodies into one"); printf("\n/CLOCK Measure and report compilation times"); @@ -318,6 +325,13 @@ extern "C" int _cdecl wmain(int argc, _In_ WCHAR **argv) else if (!_stricmp(szOpt, "DET")) { pAsm->m_fDeterministic = TRUE; +#ifdef __linux__ + if (!CryptoNative_OpenSslAvailable()) + { + fprintf(stderr, "\nWarning: OpenSSL not available. Disabling build determinism.\n"); + pAsm->m_fDeterministic = FALSE; + } +#endif } else if (!_stricmp(szOpt, "X64")) { From f3fbd133f3ec66195176dee9ce5780997e7dc489 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 15:21:27 +0000 Subject: [PATCH 32/69] Revert default timestamp --- src/coreclr/ilasm/writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index b90c35fecd931..f40069f16b0aa 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -88,7 +88,7 @@ HRESULT Assembler::InitMetaData() { // Default values for determinism. m_pPortablePdbWriter->SetGuid(GUID()); - m_pPortablePdbWriter->SetTimestamp(1); + m_pPortablePdbWriter->SetTimestamp(0); } } From 5dc8b2a0b2299bcf12dce694defb9ae6df05e4eb Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 15:22:43 +0000 Subject: [PATCH 33/69] Error checking in Apple sha256 impl --- src/coreclr/md/enc/sha256.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index f3a8acc43b225..164d6bb5af8dc 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -73,22 +73,26 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - if (dstSize < CC_SHA256_DIGEST_LENGTH) + CC_SHA256_CTX ctx = {{ 0 }}; + + if (!CC_SHA256_Init(&ctx)) { return E_FAIL; } - // Apple's documentation states these functions return 1 on success. - CC_SHA256_CTX ctx = {{ 0 }}; - int ret = CC_SHA256_Init(&ctx); - assert(ret == 1); + if (!CC_SHA256_Update(&ctx, pSrc, srcSize)) + { + return E_FAIL; + } - ret = CC_SHA256_Update(&ctx, pSrc, srcSize); - assert(ret == 1); + BYTE hash[CC_SHA256_DIGEST_LENGTH]; - ret = CC_SHA256_Final(pDst, &ctx); - assert(ret == 1); + if (!CC_SHA256_Final(hash, &ctx)) + { + return E_FAIL; + } + memcpy(pDst, hash, min(CC_SHA256_DIGEST_LENGTH, dstSize)); return S_OK; } #else From dd1161cbb6a063916aa98542c2cd2e8e96e4e187 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 16:39:57 +0000 Subject: [PATCH 34/69] Linux hash impl --- src/coreclr/ilasm/assem.cpp | 16 ++++++++++++- src/coreclr/ilasm/main.cpp | 13 ----------- src/coreclr/md/enc/CMakeLists.txt | 13 +++++++++++ src/coreclr/md/enc/sha256.cpp | 39 +++++++++++++++++++++++++++++-- src/coreclr/md/inc/sha256.h | 4 ++++ 5 files changed, 69 insertions(+), 16 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index 14e66d08f0517..d6d141e7e3950 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -15,6 +15,10 @@ #include "assembler.h" +#ifdef __linux__ +#include "sha256.h" +#endif + void indexKeywords(Indx* indx); // defined in asmparse.y unsigned int g_uCodePage = CP_ACP; @@ -242,7 +246,17 @@ BOOL Assembler::Init(BOOL generatePdb) if (m_fDeterministic) { - m_dwCeeFileFlags |= ICEE_CREATE_FILE_DET; +#ifdef __linux__ + if (!IsOpenSslAvailable()) + { + fprintf(stderr, "\nWarning: OpenSSL not available. Disabling build determinism.\n"); + m_fDeterministic = FALSE; + } + else +#endif + { + m_dwCeeFileFlags |= ICEE_CREATE_FILE_DET; + } } if (FAILED(m_pCeeFileGen->CreateCeeFileEx(&m_pCeeFile,(ULONG)m_dwCeeFileFlags))) return FALSE; diff --git a/src/coreclr/ilasm/main.cpp b/src/coreclr/ilasm/main.cpp index 28bede0b3975d..e383ca2f4bdd1 100644 --- a/src/coreclr/ilasm/main.cpp +++ b/src/coreclr/ilasm/main.cpp @@ -5,12 +5,6 @@ // -#ifdef __linux__ -extern "C" { - #include "openssl.h" -} -#endif - #include "ilasmpch.h" #include "asmparse.h" @@ -325,13 +319,6 @@ extern "C" int _cdecl wmain(int argc, _In_ WCHAR **argv) else if (!_stricmp(szOpt, "DET")) { pAsm->m_fDeterministic = TRUE; -#ifdef __linux__ - if (!CryptoNative_OpenSslAvailable()) - { - fprintf(stderr, "\nWarning: OpenSSL not available. Disabling build determinism.\n"); - pAsm->m_fDeterministic = FALSE; - } -#endif } else if (!_stricmp(szOpt, "X64")) { diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index cc9aa83d57825..d0b2e1fdacf8b 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -42,6 +42,15 @@ set(MDRUNTIMERW_HEADERS ../runtime/mdinternalro.h ) +if(CLR_CMAKE_TARGET_LINUX) + configure_file( + ../../../native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in + ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h) + include_directories(../../../native/libs/Common) + include_directories(../../../native/libs/System.Security.Cryptography.Native) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) +endif(CLR_CMAKE_TARGET_LINUX) + if (CLR_CMAKE_TARGET_WIN32) list(APPEND MDRUNTIMERW_SOURCES ${MDRUNTIMERW_HEADERS}) endif(CLR_CMAKE_TARGET_WIN32) @@ -57,6 +66,10 @@ add_library_clr(mdruntimerw_wks OBJECT ${MDRUNTIMERW_SOURCES}) target_compile_definitions(mdruntimerw_wks PRIVATE FEATURE_METADATA_EMIT_ALL) target_precompile_headers(mdruntimerw_wks PRIVATE stdafx.h) +if(CLR_CMAKE_TARGET_LINUX) + target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) +endif(CLR_CMAKE_TARGET_LINUX) + add_library_clr(mdruntimerw-dbi ${MDRUNTIMERW_SOURCES}) set_target_properties(mdruntimerw-dbi PROPERTIES DBI_COMPONENT TRUE) target_precompile_headers(mdruntimerw-dbi PRIVATE stdafx.h) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index 164d6bb5af8dc..5bc6e89f487ac 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -95,11 +95,46 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) memcpy(pDst, hash, min(CC_SHA256_DIGEST_LENGTH, dstSize)); return S_OK; } +#elif defined(__linux__) +extern "C" { + #include "openssl.h" + #include "pal_evp.h" +} + +bool IsOpenSslAvailable() +{ + return CryptoNative_OpenSslAvailable() != 0; +} + +HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +{ + if (!IsOpenSslAvailable()) + { + return E_FAIL; + } + + BYTE hash[32]; + DWORD hashLength = 0; + + if (!CryptoNative_EvpDigestOneShot(CryptoNative_EvpSha256(), pSrc, srcSize, hash, &hashLength)) + { + return E_FAIL; + } + + for (int i = 0; i < 32; i++) + { + fprintf(stderr, "%c", hash[i]); + } + + fprintf(stderr, "\n"); + + memcpy(pDst, hash, min(hashLength, dstSize)); + return S_OK; +} #else // Unsupported platform HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - memset(pDst, 0, dstSize); - return S_OK; + return E_FAIL; } #endif diff --git a/src/coreclr/md/inc/sha256.h b/src/coreclr/md/inc/sha256.h index 6374dde7316a8..b39139a9f8894 100644 --- a/src/coreclr/md/inc/sha256.h +++ b/src/coreclr/md/inc/sha256.h @@ -13,4 +13,8 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize); +#ifdef __linux__ +bool IsOpenSslAvailable(); +#endif + #endif // __sha256__h__ From 11e02e771ab4c05cbc5c246b84dbcc62cdbf2c2c Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 16:51:59 +0000 Subject: [PATCH 35/69] Move static dep to sha256 impl --- src/coreclr/ilasm/CMakeLists.txt | 15 --------------- src/coreclr/md/enc/CMakeLists.txt | 10 ++++++---- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/coreclr/ilasm/CMakeLists.txt b/src/coreclr/ilasm/CMakeLists.txt index 5321371b2af64..cca2c6da1858e 100644 --- a/src/coreclr/ilasm/CMakeLists.txt +++ b/src/coreclr/ilasm/CMakeLists.txt @@ -10,15 +10,6 @@ add_definitions(-DFEATURE_CORECLR) include_directories(.) -if(CLR_CMAKE_TARGET_LINUX) - configure_file( - ../../native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in - ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h) - include_directories(../../native/libs/Common) - include_directories(../../native/libs/System.Security.Cryptography.Native) - include_directories(${CMAKE_CURRENT_BINARY_DIR}) -endif(CLR_CMAKE_TARGET_LINUX) - set(ILASM_SOURCES assem.cpp writer.cpp @@ -117,12 +108,6 @@ else() ) endif(CLR_CMAKE_TARGET_WIN32) -if(CLR_CMAKE_TARGET_LINUX) - list(APPEND ILASM_LINK_LIBRARIES - System.Security.Cryptography.Native.OpenSsl-Static - ) -endif(CLR_CMAKE_TARGET_LINUX) - if(CLR_CMAKE_HOST_UNIX) target_link_libraries(ilasm diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index d0b2e1fdacf8b..c48f9c0600bf9 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -66,10 +66,6 @@ add_library_clr(mdruntimerw_wks OBJECT ${MDRUNTIMERW_SOURCES}) target_compile_definitions(mdruntimerw_wks PRIVATE FEATURE_METADATA_EMIT_ALL) target_precompile_headers(mdruntimerw_wks PRIVATE stdafx.h) -if(CLR_CMAKE_TARGET_LINUX) - target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) -endif(CLR_CMAKE_TARGET_LINUX) - add_library_clr(mdruntimerw-dbi ${MDRUNTIMERW_SOURCES}) set_target_properties(mdruntimerw-dbi PROPERTIES DBI_COMPONENT TRUE) target_precompile_headers(mdruntimerw-dbi PRIVATE stdafx.h) @@ -77,3 +73,9 @@ target_precompile_headers(mdruntimerw-dbi PRIVATE stdafx.h) add_library_clr(mdruntimerw_ppdb ${MDRUNTIMERW_SOURCES}) target_compile_definitions(mdruntimerw_ppdb PRIVATE FEATURE_METADATA_EMIT_ALL FEATURE_METADATA_EMIT_PORTABLE_PDB) target_precompile_headers(mdruntimerw_ppdb PRIVATE stdafx.h) + +if(CLR_CMAKE_TARGET_LINUX) + target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) + target_link_libraries(mdruntimerw_ppdb PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) +endif(CLR_CMAKE_TARGET_LINUX) + From 984f3239d78bac90bea053d6d495757bec502fe1 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 16:57:08 +0000 Subject: [PATCH 36/69] whitespace --- src/coreclr/md/enc/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index c48f9c0600bf9..a62a4e81689b7 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -78,4 +78,3 @@ if(CLR_CMAKE_TARGET_LINUX) target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) target_link_libraries(mdruntimerw_ppdb PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) endif(CLR_CMAKE_TARGET_LINUX) - From 9665944e73649a9346d48d233ec2e3c3185c16ba Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 16:52:51 +0000 Subject: [PATCH 37/69] Enable determinism tests --- src/tests/Common/CLRTest.Jit.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 3c2bf30bf913f..b0fa17b34b1dc 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -381,7 +381,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - test_det = False + test_det = True # Test determinism if test_det: From dba93cade48d5e64960a59f90da49ac7740dc267 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 17:40:25 +0000 Subject: [PATCH 38/69] Remove debug code; fix macOS build --- src/coreclr/md/enc/sha256.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index 5bc6e89f487ac..6a154e7d7dbd3 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -92,7 +92,15 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) return E_FAIL; } - memcpy(pDst, hash, min(CC_SHA256_DIGEST_LENGTH, dstSize)); + if (dstSize < CC_SHA256_DIGEST_LENGTH) + { + memcpy(pDst, hash, dstSize); + } + else + { + memcpy(pDst, hash, CC_SHA256_DIGEST_LENGTH); + } + return S_OK; } #elif defined(__linux__) @@ -121,13 +129,6 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) return E_FAIL; } - for (int i = 0; i < 32; i++) - { - fprintf(stderr, "%c", hash[i]); - } - - fprintf(stderr, "\n"); - memcpy(pDst, hash, min(hashLength, dstSize)); return S_OK; } From 52c29fd699879894eb132e868af0b655d26b4b3f Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 19:37:00 +0000 Subject: [PATCH 39/69] Fix source build --- src/coreclr/md/enc/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index a62a4e81689b7..8af638556b636 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -75,6 +75,12 @@ target_compile_definitions(mdruntimerw_ppdb PRIVATE FEATURE_METADATA_EMIT_ALL FE target_precompile_headers(mdruntimerw_ppdb PRIVATE stdafx.h) if(CLR_CMAKE_TARGET_LINUX) - target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) - target_link_libraries(mdruntimerw_ppdb PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) + if(NOT FEATURE_DISTRO_AGNOSTIC_SSL) + find_package(OpenSSL REQUIRED) + target_link_libraries(mdruntimerw_wks PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) + target_link_libraries(mdruntimerw_ppdb PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) + else() + target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) + target_link_libraries(mdruntimerw_ppdb PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) + endif() endif(CLR_CMAKE_TARGET_LINUX) From 8802b8d921b66642c3d491ef5252365e6d9d1085 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 24 Oct 2024 19:56:43 +0000 Subject: [PATCH 40/69] Remove OpenSSL find requirement --- src/coreclr/md/enc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index 8af638556b636..6c3e4cf49c8bf 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -76,7 +76,7 @@ target_precompile_headers(mdruntimerw_ppdb PRIVATE stdafx.h) if(CLR_CMAKE_TARGET_LINUX) if(NOT FEATURE_DISTRO_AGNOSTIC_SSL) - find_package(OpenSSL REQUIRED) + find_package(OpenSSL) target_link_libraries(mdruntimerw_wks PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) target_link_libraries(mdruntimerw_ppdb PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) else() From 80b4fa45166f21fce60e054f955899ae25da0347 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Thu, 24 Oct 2024 18:33:14 -0400 Subject: [PATCH 41/69] Fix building linuxdac on win --- src/coreclr/md/enc/CMakeLists.txt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index 6c3e4cf49c8bf..38c009b262de9 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -75,12 +75,14 @@ target_compile_definitions(mdruntimerw_ppdb PRIVATE FEATURE_METADATA_EMIT_ALL FE target_precompile_headers(mdruntimerw_ppdb PRIVATE stdafx.h) if(CLR_CMAKE_TARGET_LINUX) - if(NOT FEATURE_DISTRO_AGNOSTIC_SSL) - find_package(OpenSSL) - target_link_libraries(mdruntimerw_wks PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) - target_link_libraries(mdruntimerw_ppdb PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) - else() + if(FEATURE_DISTRO_AGNOSTIC_SSL) target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) target_link_libraries(mdruntimerw_ppdb PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) - endif() + else() + find_package(OpenSSL QUIET) + if(OPENSSL_FOUND) + target_link_libraries(mdruntimerw_wks PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) + target_link_libraries(mdruntimerw_ppdb PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) + endif(OPENSSL_FOUND) + endif(FEATURE_DISTRO_AGNOSTIC_SSL) endif(CLR_CMAKE_TARGET_LINUX) From f8272ffde1d326e9fcc605151c33903bb6ab5568 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Thu, 31 Oct 2024 21:22:34 +0000 Subject: [PATCH 42/69] Ensure OpenSSL is initialized --- src/coreclr/md/enc/sha256.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index 6a154e7d7dbd3..c2954ed3ba7e8 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -116,7 +116,7 @@ bool IsOpenSslAvailable() HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - if (!IsOpenSslAvailable()) + if (!IsOpenSslAvailable() || CryptoNative_EnsureOpenSslInitialized()) { return E_FAIL; } From 0e06b0a78a815d4f58e07d1cc0d6b073228653ea Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 1 Nov 2024 16:46:53 +0000 Subject: [PATCH 43/69] Exclude incompatible test from ILAsm roundtrip --- .../classloader/MethodImpl/InternalMethodImplTest.csproj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj b/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj index 8cbf249fc609e..a175bcdd6c898 100644 --- a/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj +++ b/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj @@ -5,4 +5,9 @@ - \ No newline at end of file + + + true + true + + From 097067d50b324ef8de23da15f94a27f826fb51d7 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 1 Nov 2024 18:32:48 +0000 Subject: [PATCH 44/69] Revert "Exclude incompatible test from ILAsm roundtrip" This reverts commit 0e06b0a78a815d4f58e07d1cc0d6b073228653ea. --- .../classloader/MethodImpl/InternalMethodImplTest.csproj | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj b/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj index a175bcdd6c898..8cbf249fc609e 100644 --- a/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj +++ b/src/tests/Loader/classloader/MethodImpl/InternalMethodImplTest.csproj @@ -5,9 +5,4 @@ - - - true - true - - + \ No newline at end of file From 9bfcb7f3e6d161e44589c3d299eab009d5476c80 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 1 Nov 2024 18:38:37 +0000 Subject: [PATCH 45/69] Run determinism tests only if build succeeded --- src/tests/Common/CLRTest.Jit.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index b0fa17b34b1dc..ab793f12ef06f 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -381,7 +381,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - test_det = True + test_det = proc.returncode == 0 # Test determinism if test_det: From 0940674738b13a04ec305a7ee6a2f75a70440e6d Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Mon, 4 Nov 2024 15:35:29 -0500 Subject: [PATCH 46/69] File header timestamp of 1 --- src/coreclr/dlls/mscorpe/pewriter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/dlls/mscorpe/pewriter.cpp b/src/coreclr/dlls/mscorpe/pewriter.cpp index 6d609919a236d..90a95b83308de 100644 --- a/src/coreclr/dlls/mscorpe/pewriter.cpp +++ b/src/coreclr/dlls/mscorpe/pewriter.cpp @@ -616,7 +616,8 @@ HRESULT PEWriter::Init(PESectionMan *pFrom, DWORD createFlags) if (createFlags & ICEE_CREATE_FILE_DET) { - m_ntHeaders->FileHeader.TimeDateStamp = VAL32(0); + // A timestamp of 0 triggers asserts in the VM + m_ntHeaders->FileHeader.TimeDateStamp = VAL32(1); } // Linker version should be consistent with current VC level From 786061dc3ecadcf2356c9168209e34d86f6b804b Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Tue, 5 Nov 2024 15:19:35 +0000 Subject: [PATCH 47/69] print hashes --- src/coreclr/md/enc/sha256.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index c2954ed3ba7e8..0d1088d94f8e6 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -129,6 +129,15 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) return E_FAIL; } + fprintf(stderr, "\n"); + + for (int i = 0; i < hashLength; i++) + { + fprintf(stderr, "%c", hash[i]); + } + + fprintf(stderr, "\n"); + memcpy(pDst, hash, min(hashLength, dstSize)); return S_OK; } From 6780404c7a0ced64670ac2d0d2a0929b1145ad37 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 5 Nov 2024 10:39:28 -0500 Subject: [PATCH 48/69] Fix int cmp --- src/coreclr/md/enc/sha256.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index 0d1088d94f8e6..5e3b3fc6b1fc2 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -131,7 +131,7 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) fprintf(stderr, "\n"); - for (int i = 0; i < hashLength; i++) + for (DWORD i = 0; i < hashLength; i++) { fprintf(stderr, "%c", hash[i]); } From b2b92a76bbba6f386b5e9c9a17a4ab66f40a0bba Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 5 Nov 2024 16:20:31 -0500 Subject: [PATCH 49/69] One-shot CommonCrypto SHA-256 --- src/coreclr/md/enc/sha256.cpp | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index 5e3b3fc6b1fc2..ef6a55571970f 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -73,24 +73,8 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { - CC_SHA256_CTX ctx = {{ 0 }}; - - if (!CC_SHA256_Init(&ctx)) - { - return E_FAIL; - } - - if (!CC_SHA256_Update(&ctx, pSrc, srcSize)) - { - return E_FAIL; - } - - BYTE hash[CC_SHA256_DIGEST_LENGTH]; - - if (!CC_SHA256_Final(hash, &ctx)) - { - return E_FAIL; - } + BYTE hash[32]; + CC_SHA256(pSrc, (CC_LONG)srcSize, hash); if (dstSize < CC_SHA256_DIGEST_LENGTH) { From 41c6e270b1a02f4835ada34729522e7afeaa200c Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 5 Nov 2024 16:21:53 -0500 Subject: [PATCH 50/69] Remove debug logic --- src/coreclr/md/enc/sha256.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/md/enc/sha256.cpp index ef6a55571970f..f2c3fd64bd78e 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/md/enc/sha256.cpp @@ -113,15 +113,6 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) return E_FAIL; } - fprintf(stderr, "\n"); - - for (DWORD i = 0; i < hashLength; i++) - { - fprintf(stderr, "%c", hash[i]); - } - - fprintf(stderr, "\n"); - memcpy(pDst, hash, min(hashLength, dstSize)); return S_OK; } From 047da9b15122ab6b1c23cec99e48f43ace3a473c Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Tue, 5 Nov 2024 22:15:10 +0000 Subject: [PATCH 51/69] Move SHA-256 impl to ilasm --- src/coreclr/ilasm/CMakeLists.txt | 20 +++++++++++ .../{md/enc/sha256.cpp => ilasm/sha256.h} | 36 ++++++++++++++----- src/coreclr/md/enc/CMakeLists.txt | 24 ------------- src/coreclr/md/enc/pdbheap.cpp | 3 +- src/coreclr/md/inc/sha256.h | 20 ----------- 5 files changed, 49 insertions(+), 54 deletions(-) rename src/coreclr/{md/enc/sha256.cpp => ilasm/sha256.h} (78%) delete mode 100644 src/coreclr/md/inc/sha256.h diff --git a/src/coreclr/ilasm/CMakeLists.txt b/src/coreclr/ilasm/CMakeLists.txt index 818171c25c749..8ac4ce4908105 100644 --- a/src/coreclr/ilasm/CMakeLists.txt +++ b/src/coreclr/ilasm/CMakeLists.txt @@ -34,8 +34,18 @@ set(ILASM_HEADERS nvpair.h typar.hpp portable_pdb.h + sha256.h ) +if(CLR_CMAKE_TARGET_LINUX) + configure_file( + ../../native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in + ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h) + include_directories(../../native/libs/Common) + include_directories(../../native/libs/System.Security.Cryptography.Native) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) +endif(CLR_CMAKE_TARGET_LINUX) + if(CLR_CMAKE_TARGET_WIN32) list(APPEND ILASM_SOURCES ${ILASM_HEADERS}) @@ -107,6 +117,16 @@ else() ) endif(CLR_CMAKE_TARGET_WIN32) +if(CLR_CMAKE_TARGET_LINUX) + list(APPEND ILASM_LINK_LIBRARIES System.Security.Cryptography.Native.OpenSsl-Static) + if(NOT FEATURE_DISTRO_AGNOSTIC_SSL) + find_package(OpenSSL QUIET) + if(OPENSSL_FOUND) + list(APPEND ILASM_LINK_LIBRARIES OpenSSL::SSL OpenSSL::Crypto) + endif(OPENSSL_FOUND) + endif(NOT FEATURE_DISTRO_AGNOSTIC_SSL) +endif(CLR_CMAKE_TARGET_LINUX) + if(CLR_CMAKE_HOST_UNIX) target_link_libraries(ilasm diff --git a/src/coreclr/md/enc/sha256.cpp b/src/coreclr/ilasm/sha256.h similarity index 78% rename from src/coreclr/md/enc/sha256.cpp rename to src/coreclr/ilasm/sha256.h index f2c3fd64bd78e..2ba714b57c37c 100644 --- a/src/coreclr/md/enc/sha256.cpp +++ b/src/coreclr/ilasm/sha256.h @@ -1,16 +1,18 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. //***************************************************************************** -// sha256.cpp +// sha256.h // // // contains implementation of sha256 hash algorithm // //***************************************************************************** +#ifndef __sha256__h__ +#define __sha256__h__ #ifdef _WIN32 -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +inline HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { NTSTATUS status; @@ -53,7 +55,15 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) goto cleanup; } - memcpy(pDst, hash, min(hashLength, dstSize)); + if (dstSize < hashLength) + { + memcpy(pDst, hash, dstSize); + } + else + { + memcpy(pDst, hash, hashLength); + } + status = S_OK; cleanup: @@ -71,7 +81,7 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) #include #include -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +inline HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { BYTE hash[32]; CC_SHA256(pSrc, (CC_LONG)srcSize, hash); @@ -93,12 +103,12 @@ extern "C" { #include "pal_evp.h" } -bool IsOpenSslAvailable() +inline bool IsOpenSslAvailable() { return CryptoNative_OpenSslAvailable() != 0; } -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +inline HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { if (!IsOpenSslAvailable() || CryptoNative_EnsureOpenSslInitialized()) { @@ -113,13 +123,23 @@ HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) return E_FAIL; } - memcpy(pDst, hash, min(hashLength, dstSize)); + if (dstSize < hashLength) + { + memcpy(pDst, hash, dstSize); + } + else + { + memcpy(pDst, hash, hashLength); + } + return S_OK; } #else // Unsupported platform -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) +inline HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize) { return E_FAIL; } #endif + +#endif // __sha256__h__ diff --git a/src/coreclr/md/enc/CMakeLists.txt b/src/coreclr/md/enc/CMakeLists.txt index 38c009b262de9..5b96ed8d87c97 100644 --- a/src/coreclr/md/enc/CMakeLists.txt +++ b/src/coreclr/md/enc/CMakeLists.txt @@ -4,7 +4,6 @@ set(MDRUNTIMERW_SOURCES metamodelrw.cpp peparse.cpp rwutil.cpp - sha256.cpp stgio.cpp stgtiggerstorage.cpp stgtiggerstream.cpp @@ -34,7 +33,6 @@ set(MDRUNTIMERW_HEADERS ../inc/portablepdbmdds.h ../inc/portablepdbmdi.h ../inc/rwutil.h - ../inc/sha256.h ../inc/stgio.h ../inc/stgtiggerstorage.h ../inc/stgtiggerstream.h @@ -42,15 +40,6 @@ set(MDRUNTIMERW_HEADERS ../runtime/mdinternalro.h ) -if(CLR_CMAKE_TARGET_LINUX) - configure_file( - ../../../native/libs/System.Security.Cryptography.Native/pal_crypto_config.h.in - ${CMAKE_CURRENT_BINARY_DIR}/pal_crypto_config.h) - include_directories(../../../native/libs/Common) - include_directories(../../../native/libs/System.Security.Cryptography.Native) - include_directories(${CMAKE_CURRENT_BINARY_DIR}) -endif(CLR_CMAKE_TARGET_LINUX) - if (CLR_CMAKE_TARGET_WIN32) list(APPEND MDRUNTIMERW_SOURCES ${MDRUNTIMERW_HEADERS}) endif(CLR_CMAKE_TARGET_WIN32) @@ -73,16 +62,3 @@ target_precompile_headers(mdruntimerw-dbi PRIVATE stdafx.h) add_library_clr(mdruntimerw_ppdb ${MDRUNTIMERW_SOURCES}) target_compile_definitions(mdruntimerw_ppdb PRIVATE FEATURE_METADATA_EMIT_ALL FEATURE_METADATA_EMIT_PORTABLE_PDB) target_precompile_headers(mdruntimerw_ppdb PRIVATE stdafx.h) - -if(CLR_CMAKE_TARGET_LINUX) - if(FEATURE_DISTRO_AGNOSTIC_SSL) - target_link_libraries(mdruntimerw_wks PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) - target_link_libraries(mdruntimerw_ppdb PRIVATE System.Security.Cryptography.Native.OpenSsl-Static) - else() - find_package(OpenSSL QUIET) - if(OPENSSL_FOUND) - target_link_libraries(mdruntimerw_wks PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) - target_link_libraries(mdruntimerw_ppdb PRIVATE OpenSSL::SSL OpenSSL::Crypto System.Security.Cryptography.Native.OpenSsl-Static) - endif(OPENSSL_FOUND) - endif(FEATURE_DISTRO_AGNOSTIC_SSL) -endif(CLR_CMAKE_TARGET_LINUX) diff --git a/src/coreclr/md/enc/pdbheap.cpp b/src/coreclr/md/enc/pdbheap.cpp index a575354b9d541..81a1fff8a6eee 100644 --- a/src/coreclr/md/enc/pdbheap.cpp +++ b/src/coreclr/md/enc/pdbheap.cpp @@ -3,7 +3,6 @@ #include "stdafx.h" #include "pdbheap.h" -#include "sha256.h" PdbHeap::PdbHeap() : m_data(NULL), m_size(0) { @@ -83,7 +82,7 @@ __checkReturn HRESULT PdbHeap::ComputeSha256Checksum(BYTE (&checksum)[32]) { _ASSERTE(m_size >= sizeof(PDB_ID)); - return Sha256Hash(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); + return E_FAIL; // Sha256Hash(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); } __checkReturn diff --git a/src/coreclr/md/inc/sha256.h b/src/coreclr/md/inc/sha256.h deleted file mode 100644 index b39139a9f8894..0000000000000 --- a/src/coreclr/md/inc/sha256.h +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -//***************************************************************************** -// sha256.h -// - -// -// contains implementation of sha256 hash algorithm -// -//***************************************************************************** -#ifndef __sha256__h__ -#define __sha256__h__ - -HRESULT Sha256Hash(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize); - -#ifdef __linux__ -bool IsOpenSslAvailable(); -#endif - -#endif // __sha256__h__ From 1bf147cf0c58651839647d5083889391194e5200 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Tue, 5 Nov 2024 22:23:59 +0000 Subject: [PATCH 52/69] Add hash function arg to ComputeSha256Checksum --- src/coreclr/ilasm/portable_pdb.cpp | 3 ++- src/coreclr/md/compiler/emit.cpp | 3 ++- src/coreclr/md/compiler/regmeta.h | 1 + src/coreclr/md/enc/pdbheap.cpp | 4 ++-- src/coreclr/md/inc/pdbheap.h | 2 +- src/coreclr/md/inc/portablepdbmdi.h | 1 + 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/coreclr/ilasm/portable_pdb.cpp b/src/coreclr/ilasm/portable_pdb.cpp index 80c8d20440b29..a60214c99e9f3 100644 --- a/src/coreclr/ilasm/portable_pdb.cpp +++ b/src/coreclr/ilasm/portable_pdb.cpp @@ -4,6 +4,7 @@ #include "portable_pdb.h" #include #include "assembler.h" +#include "sha256.h" //***************************************************************************** // Document @@ -161,7 +162,7 @@ HRESULT PortablePdbWriter::BuildPdbStream(IMetaDataEmit3* peEmitter, mdMethodDef HRESULT PortablePdbWriter::ComputeSha256PdbStreamChecksum(BYTE(&checksum)[32]) { - return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(checksum); + return m_ilasmPdbWriter->ComputeSha256PdbStreamChecksum(Sha256Hash, checksum); } HRESULT PortablePdbWriter::ChangePdbStreamGuid(REFGUID newGuid) diff --git a/src/coreclr/md/compiler/emit.cpp b/src/coreclr/md/compiler/emit.cpp index 600e4f11898de..3c6dae259e474 100644 --- a/src/coreclr/md/compiler/emit.cpp +++ b/src/coreclr/md/compiler/emit.cpp @@ -2119,12 +2119,13 @@ STDMETHODIMP RegMeta::DefineLocalVariable( // S_OK or error. // ComputeSha256PdbStreamChecksum //******************************************************************************* STDMETHODIMP RegMeta::ComputeSha256PdbStreamChecksum( + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]) { #ifdef FEATURE_METADATA_EMIT_IN_DEBUGGER return E_NOTIMPL; #else //!FEATURE_METADATA_EMIT_IN_DEBUGGER - return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(checksum); + return m_pStgdb->m_pPdbHeap->ComputeSha256Checksum(computeSha256, checksum); #endif //!FEATURE_METADATA_EMIT_IN_DEBUGGER } diff --git a/src/coreclr/md/compiler/regmeta.h b/src/coreclr/md/compiler/regmeta.h index bcbc41fc479c1..925be2775144c 100644 --- a/src/coreclr/md/compiler/regmeta.h +++ b/src/coreclr/md/compiler/regmeta.h @@ -1138,6 +1138,7 @@ class RegMeta : // IILAsmPortablePdbWriter methods //***************************************************************************** STDMETHODIMP ComputeSha256PdbStreamChecksum( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] BYTE (&checksum)[32]); // [OUT] 256-bit Pdb checksum STDMETHODIMP ChangePdbStreamGuid( // S_OK or error. diff --git a/src/coreclr/md/enc/pdbheap.cpp b/src/coreclr/md/enc/pdbheap.cpp index 81a1fff8a6eee..d402cddc0a731 100644 --- a/src/coreclr/md/enc/pdbheap.cpp +++ b/src/coreclr/md/enc/pdbheap.cpp @@ -79,10 +79,10 @@ HRESULT PdbHeap::SetDataGuid(REFGUID newGuid) } __checkReturn -HRESULT PdbHeap::ComputeSha256Checksum(BYTE (&checksum)[32]) +HRESULT PdbHeap::ComputeSha256Checksum(HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]) { _ASSERTE(m_size >= sizeof(PDB_ID)); - return E_FAIL; // Sha256Hash(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); + return computeSha256(m_data, m_size, (BYTE*)&checksum, sizeof(checksum)); } __checkReturn diff --git a/src/coreclr/md/inc/pdbheap.h b/src/coreclr/md/inc/pdbheap.h index 117a07fe0b354..88bae893f86d7 100644 --- a/src/coreclr/md/inc/pdbheap.h +++ b/src/coreclr/md/inc/pdbheap.h @@ -22,7 +22,7 @@ class PdbHeap __checkReturn HRESULT SetData(PORT_PDB_STREAM* data); __checkReturn HRESULT SetDataGuid(REFGUID newGuid); - __checkReturn HRESULT ComputeSha256Checksum(BYTE (&checksum)[32]); + __checkReturn HRESULT ComputeSha256Checksum(HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), BYTE (&checksum)[32]); __checkReturn HRESULT SaveToStream(IStream* stream); BOOL IsEmpty(); ULONG GetSize(); diff --git a/src/coreclr/md/inc/portablepdbmdi.h b/src/coreclr/md/inc/portablepdbmdi.h index a05bc163c2876..fffdfbe95a5ff 100644 --- a/src/coreclr/md/inc/portablepdbmdi.h +++ b/src/coreclr/md/inc/portablepdbmdi.h @@ -101,6 +101,7 @@ EXTERN_GUID(IID_IILAsmPortablePdbWriter, 0x8b2db1f0, 0x91f5, 0x4c99, 0xbb, 0x07, DECLARE_INTERFACE_(IILAsmPortablePdbWriter, IUnknown) { STDMETHOD(ComputeSha256PdbStreamChecksum)( // S_OK or error. + HRESULT (*computeSha256)(BYTE* pSrc, DWORD srcSize, BYTE* pDst, DWORD dstSize), // [IN] BYTE (&checksum)[32]) PURE; // [OUT] 256-bit Pdb checksum STDMETHOD(ChangePdbStreamGuid)( // S_OK or error. From 8214e8d92aa33c1e8b2d75a910eae1dc14c99abe Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Thu, 7 Nov 2024 11:03:13 -0500 Subject: [PATCH 53/69] Fix hash call site --- src/coreclr/ilasm/writer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index f40069f16b0aa..960624ac68b06 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1703,9 +1703,12 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) // created in Assembler::InitMetaData, and it is guaranteed that the IMDInternalEmit for // that scope was already acquired immediately after that scope was created. _ASSERTE(m_pInternalEmitForDeterministicMvid != NULL); + _ASSERTE(sizeof(GUID) <= 32); + BYTE hash[32]; + if (FAILED(hr = Sha256Hash(metaData, metaDataSize, hash, sizeof(hash)))) goto exit; + GUID mvid; - hr = Sha256Hash(metaData, metaDataSize, (BYTE*)&mvid, sizeof(GUID)); - if (FAILED(hr)) goto exit; + memcpy(&mvid, hash, sizeof(GUID)); m_pInternalEmitForDeterministicMvid->ChangeMvid(mvid); } From ca3edb1d42b72cf487d116b969e87284f34ca52d Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Thu, 7 Nov 2024 13:52:15 -0500 Subject: [PATCH 54/69] Tweak determinism test logic --- src/tests/Common/CLRTest.Jit.targets | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index ab793f12ef06f..e8eca0d4cec1f 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -300,12 +300,11 @@ import glob import hashlib def hash_file(filename): - h = hashlib.sha256() - b = bytearray(128*1024) - mv = memoryview(b) - with open(filename, 'rb', buffering=0) as f: - for n in iter(lambda : f.readinto(mv), 0): - h.update(mv[:n]) + h = hashlib.sha256() + BUF_SIZE = 128*1024 + with open(filename, "rb", buffering=0) as f: + for chunk in iter(lambda : f.read(BUF_SIZE), b""): + h.update(chunk) return h.hexdigest() def is_managed_assembly(file): @@ -386,6 +385,7 @@ for inputAssemblyName in glob.glob("*.dll"): # Test determinism if test_det: + print(f"file1 size: {os.path.getsize(inputAssemblyName)}") hash = hash_file(inputAssemblyName) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' @@ -398,6 +398,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) + print(f"file2 size: {os.path.getsize(inputAssemblyName)}") if hash != hash_file(inputAssemblyName): print("ILASM determinism failed") sys.exit(1) From ea562ae37d147e93ed1b0bc8aa8439d90dff339c Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Fri, 8 Nov 2024 16:11:59 -0500 Subject: [PATCH 55/69] Revert "Tweak determinism test logic" This reverts commit ca3edb1d42b72cf487d116b969e87284f34ca52d. --- src/tests/Common/CLRTest.Jit.targets | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index e8eca0d4cec1f..ab793f12ef06f 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -300,11 +300,12 @@ import glob import hashlib def hash_file(filename): - h = hashlib.sha256() - BUF_SIZE = 128*1024 - with open(filename, "rb", buffering=0) as f: - for chunk in iter(lambda : f.read(BUF_SIZE), b""): - h.update(chunk) + h = hashlib.sha256() + b = bytearray(128*1024) + mv = memoryview(b) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda : f.readinto(mv), 0): + h.update(mv[:n]) return h.hexdigest() def is_managed_assembly(file): @@ -385,7 +386,6 @@ for inputAssemblyName in glob.glob("*.dll"): # Test determinism if test_det: - print(f"file1 size: {os.path.getsize(inputAssemblyName)}") hash = hash_file(inputAssemblyName) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' @@ -398,7 +398,6 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - print(f"file2 size: {os.path.getsize(inputAssemblyName)}") if hash != hash_file(inputAssemblyName): print("ILASM determinism failed") sys.exit(1) From 6d6b058e71cbac27a07f84220241bcb8fe948fe8 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Fri, 8 Nov 2024 16:18:46 -0500 Subject: [PATCH 56/69] Whitespace --- src/coreclr/ilasm/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/ilasm/CMakeLists.txt b/src/coreclr/ilasm/CMakeLists.txt index 2a15a4238f0f2..d7f5ce9a15aa8 100644 --- a/src/coreclr/ilasm/CMakeLists.txt +++ b/src/coreclr/ilasm/CMakeLists.txt @@ -127,6 +127,7 @@ if(CLR_CMAKE_TARGET_UNIX AND NOT CLR_CMAKE_TARGET_APPLE) endif(NOT FEATURE_DISTRO_AGNOSTIC_SSL) endif(CLR_CMAKE_TARGET_UNIX AND NOT CLR_CMAKE_TARGET_APPLE) + if(CLR_CMAKE_HOST_UNIX) target_link_libraries(ilasm PRIVATE From 91c2acff9b65fae57f871f5c7e8e8b2b498349fd Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Fri, 8 Nov 2024 16:20:51 -0500 Subject: [PATCH 57/69] Clean up macro --- src/coreclr/ilasm/assem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index 3c4dee5ca4590..62f3ca076ec35 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -15,7 +15,7 @@ #include "assembler.h" -#ifdef __linux__ +#if !defined(_WIN32) && !defined(__APPLE__) #include "sha256.h" #endif @@ -247,7 +247,7 @@ BOOL Assembler::Init(BOOL generatePdb) if (m_fDeterministic) { -#ifdef __linux__ +#if !defined(_WIN32) && !defined(__APPLE__) if (!IsOpenSslAvailable()) { fprintf(stderr, "\nWarning: OpenSSL not available. Disabling build determinism.\n"); From db19e8ad391f26d94170accb9aed726e1b2caa83 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Mon, 6 Jan 2025 12:25:25 -0500 Subject: [PATCH 58/69] Use valid deterministic timestamp --- src/coreclr/dlls/mscorpe/pewriter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/dlls/mscorpe/pewriter.cpp b/src/coreclr/dlls/mscorpe/pewriter.cpp index 951f3b14ef979..1232c0a542307 100644 --- a/src/coreclr/dlls/mscorpe/pewriter.cpp +++ b/src/coreclr/dlls/mscorpe/pewriter.cpp @@ -616,8 +616,8 @@ HRESULT PEWriter::Init(PESectionMan *pFrom, DWORD createFlags) if (createFlags & ICEE_CREATE_FILE_DET) { - // A timestamp of 0 triggers asserts in the VM - m_ntHeaders->FileHeader.TimeDateStamp = VAL32(1); + // We don't need a meaningful date/time stamp -- just a consistent one + m_ntHeaders->FileHeader.TimeDateStamp = VAL32(0xFFFFFFFF); } // Linker version should be consistent with current VC level From f7162b4d86b64e3d54650d5d99e0ae70837753fc Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Mon, 6 Jan 2025 13:21:14 -0500 Subject: [PATCH 59/69] Ensure SHA256 prefix to PDB checksum is null-terminated --- src/coreclr/ilasm/writer.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 960624ac68b06..5060aea370434 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -335,13 +335,15 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) // Algorithm name is case sensitive. const char* algoName = "SHA256"; - DWORD pdbChecksumSize = (DWORD)strlen(algoName) + 1 + sizeof(pdbChecksum); + const DWORD pdbChecksumOffset = ((DWORD)strlen(algoName)) + 1; + const DWORD pdbChecksumSize = pdbChecksumOffset + sizeof(pdbChecksum); BYTE* pdbChecksumData = new BYTE[pdbChecksumSize]; - DWORD pdbChecksumOffset = 0; - memcpy_s(pdbChecksumData + pdbChecksumOffset, pdbChecksumSize, algoName, strlen(algoName)); // AlgorithmName - pdbChecksumOffset += (DWORD)strlen(algoName) + 1; - memcpy_s(pdbChecksumData + pdbChecksumOffset, pdbChecksumSize, &pdbChecksum, sizeof(pdbChecksum)); // Checksum + // AlgorithmName (including null terminator) + memcpy_s(pdbChecksumData, pdbChecksumSize, algoName, pdbChecksumOffset); + + // Checksum + memcpy_s(pdbChecksumData + pdbChecksumOffset, pdbChecksumSize - pdbChecksumOffset, &pdbChecksum, sizeof(pdbChecksum)); /* END PDB CHECKSUM */ auto finish = From 2614e3643b24a40858505cc6069f5b7cdcaafae0 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 7 Jan 2025 12:16:51 -0500 Subject: [PATCH 60/69] Error out if OpenSSL not available --- src/coreclr/ilasm/assem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index da89fc94cb390..f70636e73c318 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -250,8 +250,8 @@ BOOL Assembler::Init(BOOL generatePdb) #if !defined(_WIN32) && !defined(__APPLE__) if (!IsOpenSslAvailable()) { - fprintf(stderr, "\nWarning: OpenSSL not available. Disabling build determinism.\n"); - m_fDeterministic = FALSE; + fprintf(stderr, "OpenSSL is not available, but required for build determinism\n"); + return FALSE; } else #endif From 9247c7a6e44b454934b512cdd8319df210f8cb8e Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 7 Jan 2025 13:14:00 -0500 Subject: [PATCH 61/69] Compare files instead of hashes for determinism test --- src/tests/Common/CLRTest.Jit.targets | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index ab793f12ef06f..b19e6169e4886 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -297,16 +297,7 @@ import shutil import subprocess import sys import glob -import hashlib - -def hash_file(filename): - h = hashlib.sha256() - b = bytearray(128*1024) - mv = memoryview(b) - with open(filename, 'rb', buffering=0) as f: - for n in iter(lambda : f.readinto(mv), 0): - h.update(mv[:n]) - return h.hexdigest() +import filecmp def is_managed_assembly(file): proc = subprocess.Popen([f'{os.environ["CORE_ROOT"]}/corerun', f'{os.environ["CORE_ROOT"]}/AssemblyChecker/AssemblyChecker.dll', file]) @@ -386,9 +377,7 @@ for inputAssemblyName in glob.glob("*.dll"): # Test determinism if test_det: - hash = hash_file(inputAssemblyName) - - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output=diff.{inputAssemblyName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) proc = subprocess.Popen(ilasm_args, shell=True) @@ -398,7 +387,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - if hash != hash_file(inputAssemblyName): + if not filecmp.cmp(inputAssemblyName, f'diff.{inputAssemblyName}', shallow=False): print("ILASM determinism failed") sys.exit(1) else: @@ -421,9 +410,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - hash = hash_file(pdbName) - - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output=diff.{pdbName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) proc = subprocess.Popen(ilasm_args, shell=True) @@ -433,7 +420,7 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - if hash != hash_file(pdbName): + if not filecmp.cmp(pdbName, f'diff.{pdbName}', shallow=False): print("ILASM PDB determinism failed") sys.exit(1) else: From 92ef0a9d850297f308fb368935d4f0e1d41d4ec2 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 7 Jan 2025 13:32:50 -0500 Subject: [PATCH 62/69] Computed timestamp for debug dirs from PDB checksum --- src/coreclr/ilasm/writer.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 5060aea370434..b39c1fb533a7a 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -86,7 +86,8 @@ HRESULT Assembler::InitMetaData() if (m_fDeterministic) { - // Default values for determinism. + // When build determinism is enabled, the PE file will be hashed with these fields set to zero. + // Then, the deterministic GUID and timestamp will be derived from the hash. m_pPortablePdbWriter->SetGuid(GUID()); m_pPortablePdbWriter->SetTimestamp(0); } @@ -377,7 +378,7 @@ HRESULT Assembler::CreateDebugDirectory(BYTE(&pdbChecksum)[32]) hr = addEntry( /* characteristics */ VAL32(0), - /* timeDateStamp */ VAL32(0), + /* timeDateStamp */ VAL32(m_pPortablePdbWriter->GetTimestamp()), /* majorVersion */ VAL16(1), /* minorVersion */ VAL16(0), /* type */ VAL32(/* PDB Checksum Debug Directory Entry */ 19), @@ -1469,8 +1470,14 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (m_fDeterministic) { + // Get deterministic GUID and timestamp from the computed hash + _ASSERTE(sizeof(GUID) + sizeof(ULONG) <= sizeof(pdbChecksum)); GUID pdbGuid = *((GUID*)&pdbChecksum); if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(pdbGuid))) goto exit; + + ULONG timestamp; + memcpy_s(×tamp, sizeof(ULONG), pdbChecksum + sizeof(GUID), sizeof(ULONG)); + m_pPortablePdbWriter->SetTimestamp(timestamp); } if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; From b7ad2c68e1eb7676a00ebf83185d2ad62e90f841 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 7 Jan 2025 14:14:15 -0500 Subject: [PATCH 63/69] Extend PEWriter API surface to change file header timestamp --- src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp | 5 +++++ src/coreclr/dlls/mscorpe/iceefilegen.cpp | 11 ++++++++++- src/coreclr/dlls/mscorpe/pewriter.cpp | 11 +++++------ src/coreclr/dlls/mscorpe/pewriter.h | 1 + src/coreclr/ilasm/assem.cpp | 16 +++++++--------- src/coreclr/ilasm/writer.cpp | 1 + src/coreclr/inc/ceefilegenwriter.h | 1 + src/coreclr/inc/iceefilegen.h | 3 +-- 8 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp b/src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp index 5edec7ed47c69..0b2b72453fbbf 100644 --- a/src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp +++ b/src/coreclr/dlls/mscorpe/ceefilegenwriter.cpp @@ -482,6 +482,11 @@ HRESULT CeeFileGenWriter::getFileTimeStamp(DWORD *pTimeStamp) return getPEWriter().getFileTimeStamp(pTimeStamp); } // HRESULT CeeFileGenWriter::getFileTimeStamp() +void CeeFileGenWriter::setFileHeaderTimeStamp(DWORD timeStamp) +{ + return getPEWriter().setFileHeaderTimeStamp(timeStamp); +} // void CeeFileGenWriter::setFileHeaderTimeStamp() + HRESULT CeeFileGenWriter::setAddrReloc(UCHAR *instrAddr, DWORD value) { *(DWORD *)instrAddr = VAL32(value); diff --git a/src/coreclr/dlls/mscorpe/iceefilegen.cpp b/src/coreclr/dlls/mscorpe/iceefilegen.cpp index 152fe9b3fcaf9..06270631c551c 100644 --- a/src/coreclr/dlls/mscorpe/iceefilegen.cpp +++ b/src/coreclr/dlls/mscorpe/iceefilegen.cpp @@ -446,7 +446,7 @@ HRESULT ICeeFileGen::SetVTableEntry64(HCEEFILE ceeFile, ULONG size, void* ptr) return gen->setVTableEntry64(size, ptr); } -HRESULT ICeeFileGen::GetFileTimeStamp (HCEEFILE ceeFile, DWORD *pTimeStamp) +HRESULT ICeeFileGen::GetFileTimeStamp(HCEEFILE ceeFile, DWORD *pTimeStamp) { TESTANDRETURNPOINTER(ceeFile); TESTANDRETURNPOINTER(pTimeStamp); @@ -455,3 +455,12 @@ HRESULT ICeeFileGen::GetFileTimeStamp (HCEEFILE ceeFile, DWORD *pTimeStamp) return(gen->getFileTimeStamp(pTimeStamp)); } +HRESULT ICeeFileGen::SetFileHeaderTimeStamp(HCEEFILE ceeFile, DWORD timeStamp) +{ + TESTANDRETURNPOINTER(ceeFile); + + CeeFileGenWriter *gen = reinterpret_cast(ceeFile); + gen->setFileHeaderTimeStamp(timeStamp); + return S_OK; +} + diff --git a/src/coreclr/dlls/mscorpe/pewriter.cpp b/src/coreclr/dlls/mscorpe/pewriter.cpp index 1232c0a542307..703c98e1a17b1 100644 --- a/src/coreclr/dlls/mscorpe/pewriter.cpp +++ b/src/coreclr/dlls/mscorpe/pewriter.cpp @@ -614,12 +614,6 @@ HRESULT PEWriter::Init(PESectionMan *pFrom, DWORD createFlags) m_ntHeaders->FileHeader.Characteristics |= VAL16(IMAGE_FILE_RELOCS_STRIPPED); } - if (createFlags & ICEE_CREATE_FILE_DET) - { - // We don't need a meaningful date/time stamp -- just a consistent one - m_ntHeaders->FileHeader.TimeDateStamp = VAL32(0xFFFFFFFF); - } - // Linker version should be consistent with current VC level m_ntHeaders->OptionalHeader.MajorLinkerVersion = 11; m_ntHeaders->OptionalHeader.MinorLinkerVersion = 0; @@ -1643,6 +1637,11 @@ HRESULT PEWriter::getFileTimeStamp(DWORD *pTimeStamp) return S_OK; } +void PEWriter::setFileHeaderTimeStamp(DWORD timeStamp) +{ + m_ntHeaders->FileHeader.TimeDateStamp = timeStamp; +} + DWORD PEWriter::getImageBase32() { _ASSERTE(isPE32()); diff --git a/src/coreclr/dlls/mscorpe/pewriter.h b/src/coreclr/dlls/mscorpe/pewriter.h index fbb2e83e1b061..31b4ddcb726eb 100644 --- a/src/coreclr/dlls/mscorpe/pewriter.h +++ b/src/coreclr/dlls/mscorpe/pewriter.h @@ -83,6 +83,7 @@ class PEWriter : public PESectionMan size_t getImageBase(); HRESULT getFileTimeStamp(DWORD *pTimeStamp); + void setFileHeaderTimeStamp(DWORD timeStamp); IMAGE_NT_HEADERS32* ntHeaders32() { return (IMAGE_NT_HEADERS32*) m_ntHeaders; } IMAGE_NT_HEADERS64* ntHeaders64() { return (IMAGE_NT_HEADERS64*) m_ntHeaders; } diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index f70636e73c318..2076192138bfe 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -244,6 +244,10 @@ BOOL Assembler::Init(BOOL generatePdb) } if (FAILED(CreateICeeFileGen(&m_pCeeFileGen))) return FALSE; + if (FAILED(m_pCeeFileGen->CreateCeeFileEx(&m_pCeeFile,(ULONG)m_dwCeeFileFlags))) return FALSE; + if (FAILED(m_pCeeFileGen->GetSectionCreate(m_pCeeFile, ".il", sdReadOnly, &m_pILSection))) return FALSE; + if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".sdata", sdReadWrite, &m_pGlobalDataSection))) return FALSE; + if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".tls", sdReadWrite, &m_pTLSSection))) return FALSE; if (m_fDeterministic) { @@ -255,17 +259,11 @@ BOOL Assembler::Init(BOOL generatePdb) } else #endif - { - m_dwCeeFileFlags |= ICEE_CREATE_FILE_DET; - } + // Initialize file header timestamp to something consistent. + // If we're going to generate a PDB, we will update this timestamp with a value computed from the PDB's hash. + if (FAILED(m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, VAL32(0xFFFFFFFF)))) return FALSE; } - if (FAILED(m_pCeeFileGen->CreateCeeFileEx(&m_pCeeFile,(ULONG)m_dwCeeFileFlags))) return FALSE; - - if (FAILED(m_pCeeFileGen->GetSectionCreate(m_pCeeFile, ".il", sdReadOnly, &m_pILSection))) return FALSE; - if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".sdata", sdReadWrite, &m_pGlobalDataSection))) return FALSE; - if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".tls", sdReadWrite, &m_pTLSSection))) return FALSE; - m_fGeneratePDB = generatePdb; return TRUE; diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index b39c1fb533a7a..de5bec7b7f845 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1478,6 +1478,7 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) ULONG timestamp; memcpy_s(×tamp, sizeof(ULONG), pdbChecksum + sizeof(GUID), sizeof(ULONG)); m_pPortablePdbWriter->SetTimestamp(timestamp); + if (FAILED(hr = m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, timestamp))) goto exit; } if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; diff --git a/src/coreclr/inc/ceefilegenwriter.h b/src/coreclr/inc/ceefilegenwriter.h index 0364767cd5f47..2acbcb4f04ead 100644 --- a/src/coreclr/inc/ceefilegenwriter.h +++ b/src/coreclr/inc/ceefilegenwriter.h @@ -108,6 +108,7 @@ class CeeFileGenWriter : public CCeeGen HRESULT getCorHeader(IMAGE_COR20_HEADER **ppHeader); HRESULT getFileTimeStamp(DWORD *pTimeStamp); + void setFileHeaderTimeStamp(DWORD timeStamp); HRESULT setLibraryGuid(_In_ LPWSTR libraryGuid); diff --git a/src/coreclr/inc/iceefilegen.h b/src/coreclr/inc/iceefilegen.h index 5a752aeba0ac4..43d92e4f2c883 100644 --- a/src/coreclr/inc/iceefilegen.h +++ b/src/coreclr/inc/iceefilegen.h @@ -66,8 +66,6 @@ typedef HRESULT (__stdcall * PFN_DestroyICeeFileGen)(ICeeFileGen ** ceeFileGen); #define ICEE_CREATE_MACHINE_ARM 0x00000400 // Create a IMAGE_FILE_MACHINE_ARMNT #define ICEE_CREATE_MACHINE_ARM64 0x00000800 // Create a IMAGE_FILE_MACHINE_ARM64 -#define ICEE_CREATE_FILE_DET 0x00100000 // Creates a deterministic PE - // Pass this to CreateCeeFileEx to create a pure IL Exe or DLL #define ICEE_CREATE_FILE_PURE_IL ICEE_CREATE_FILE_PE32 | \ ICEE_CREATE_FILE_CORMAIN_STUB | \ @@ -157,6 +155,7 @@ class ICeeFileGen { BYTE* buffer, unsigned buffLen); virtual HRESULT GetFileTimeStamp (HCEEFILE ceeFile, DWORD *pTimeStamp); + virtual HRESULT SetFileHeaderTimeStamp(HCEEFILE ceeFile, DWORD timeStamp); virtual HRESULT SetFileAlignment(HCEEFILE ceeFile, ULONG fileAlignment); From b15f8100706cbe587b0ffa4e3e346f5b87875777 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 7 Jan 2025 16:54:30 -0500 Subject: [PATCH 64/69] Fix ILAsm roundtrip tests --- src/tests/Common/CLRTest.Jit.targets | 34 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index b19e6169e4886..58860d97fe038 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -377,7 +377,13 @@ for inputAssemblyName in glob.glob("*.dll"): # Test determinism if test_det: - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output=diff.{inputAssemblyName} {ilasmSwitches} {disassemblyName}' + try: + os.rename(inputAssemblyName, f'{inputAssemblyName}.base') + except: + print("Could not find f'{inputAssemblyName}'") + sys.exit(1) + + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) proc = subprocess.Popen(ilasm_args, shell=True) @@ -387,7 +393,13 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - if not filecmp.cmp(inputAssemblyName, f'diff.{inputAssemblyName}', shallow=False): + try: + os.rename(inputAssemblyName, f'{inputAssemblyName}.diff') + except: + print("Could not find f'{inputAssemblyName}'") + sys.exit(1) + + if not filecmp.cmp(f'{inputAssemblyName}.base', f'{inputAssemblyName}.diff', shallow=False): print("ILASM determinism failed") sys.exit(1) else: @@ -395,6 +407,8 @@ for inputAssemblyName in glob.glob("*.dll"): # Test PDB determinism + os.rename(f'{inputAssemblyName}.base', inputAssemblyName) + if not is_managed_debug_assembly(inputAssemblyName): ilasmSwitches = ilasmSwitches + " -DEBUG" @@ -410,7 +424,13 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output=diff.{pdbName} {ilasmSwitches} {disassemblyName}' + try: + os.rename(pdbName, f'{pdbName}.base') + except: + print("Could not find f'{pdbName}'") + sys.exit(1) + + ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' print(ilasm_args) proc = subprocess.Popen(ilasm_args, shell=True) @@ -420,7 +440,13 @@ for inputAssemblyName in glob.glob("*.dll"): proc.kill() sys.exit(1) - if not filecmp.cmp(pdbName, f'diff.{pdbName}', shallow=False): + try: + os.rename(pdbName, f'{pdbName}.diff') + except: + print("Could not find f{pdbName}") + sys.exit(1) + + if not filecmp.cmp(f'{pdbName}.base', f'{pdbName}.diff', shallow=False): print("ILASM PDB determinism failed") sys.exit(1) else: From db1cdfee4960b8c67b714bec7bb47727c35285f1 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 7 Jan 2025 17:07:45 -0500 Subject: [PATCH 65/69] Remove comment --- src/coreclr/ilasm/writer.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index de5bec7b7f845..f8e0981bc0734 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -59,12 +59,6 @@ HRESULT Assembler::InitMetaData() // was created above, and the ChangeMvid service that makes this possible is only available // on the IMDInternalEmit interface. // - // When the CLSID_CorMetaDataDispenser instance above has activated against a current - // clr.dll (which is the only supported configuration for the determinism feature), it is - // guaranteed that "m_pEmitter" is implemented by the RegMeta object that was created - // during the DefineScope call above, and it is guaranteed that this same RegMeta object - // also implements the required IMDInternalEmit interface. - // // Any failure is unexpected and catastrophic, so print a noisy message and return an // error (which generally fails the entire ilasm operation) if any failure occurs. // From 2ce73214eed735895722c42f01d7ebab2d917ec4 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 8 Jan 2025 13:22:35 -0500 Subject: [PATCH 66/69] Don't use PDB checksum for GUID/timestamp --- src/coreclr/ilasm/assem.cpp | 17 +++------ src/coreclr/ilasm/writer.cpp | 73 +++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/coreclr/ilasm/assem.cpp b/src/coreclr/ilasm/assem.cpp index 2076192138bfe..f923c34595bcd 100644 --- a/src/coreclr/ilasm/assem.cpp +++ b/src/coreclr/ilasm/assem.cpp @@ -249,20 +249,13 @@ BOOL Assembler::Init(BOOL generatePdb) if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".sdata", sdReadWrite, &m_pGlobalDataSection))) return FALSE; if (FAILED(m_pCeeFileGen->GetSectionCreate (m_pCeeFile, ".tls", sdReadWrite, &m_pTLSSection))) return FALSE; - if (m_fDeterministic) - { #if !defined(_WIN32) && !defined(__APPLE__) - if (!IsOpenSslAvailable()) - { - fprintf(stderr, "OpenSSL is not available, but required for build determinism\n"); - return FALSE; - } - else -#endif - // Initialize file header timestamp to something consistent. - // If we're going to generate a PDB, we will update this timestamp with a value computed from the PDB's hash. - if (FAILED(m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, VAL32(0xFFFFFFFF)))) return FALSE; + if (m_fDeterministic && !IsOpenSslAvailable()) + { + fprintf(stderr, "OpenSSL is not available, but required for build determinism\n"); + return FALSE; } +#endif m_fGeneratePDB = generatePdb; diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index f8e0981bc0734..8138d7ac6d74c 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -80,8 +80,8 @@ HRESULT Assembler::InitMetaData() if (m_fDeterministic) { - // When build determinism is enabled, the PE file will be hashed with these fields set to zero. - // Then, the deterministic GUID and timestamp will be derived from the hash. + // When build determinism is enabled, the PDB checksum is computed with these fields set to zero. + // The GUID and timestamp will be updated after. m_pPortablePdbWriter->SetGuid(GUID()); m_pPortablePdbWriter->SetTimestamp(0); } @@ -1452,6 +1452,37 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr=CreateTLSDirectory())) goto exit; + // Reserve a buffer for the meta-data + DWORD metaDataSize; + if (FAILED(hr=m_pEmitter->GetSaveSize(cssAccurate, &metaDataSize))) goto exit; + BYTE* metaData; + if (FAILED(hr=m_pCeeFileGen->GetSectionBlock(m_pILSection, metaDataSize, sizeof(DWORD), (void**) &metaData))) goto exit; + ULONG metaDataOffset; + if (FAILED(hr=m_pCeeFileGen->GetSectionDataLen(m_pILSection, &metaDataOffset))) goto exit; + metaDataOffset -= metaDataSize; + + GUID deterministicGuid = GUID(); + ULONG deterministicTimestamp = 0; + + if (m_fDeterministic) + { + // Get deterministic GUID and timestamp from the computed hash + BYTE hash[32]; + _ASSERTE(sizeof(GUID) + sizeof(ULONG) <= sizeof(hash)); + if (FAILED(hr = Sha256Hash(metaData, metaDataSize, hash, sizeof(hash)))) goto exit; + + memcpy_s(&deterministicGuid, sizeof(GUID), hash, sizeof(GUID)); + memcpy_s(&deterministicTimestamp, sizeof(ULONG), hash + sizeof(GUID), sizeof(ULONG)); + + // In deterministic mode, the MVID needs to be stabilized for the metadata scope that was + // created in Assembler::InitMetaData, and it is guaranteed that the IMDInternalEmit for + // that scope was already acquired immediately after that scope was created. + _ASSERTE(m_pInternalEmitForDeterministicMvid != NULL); + m_pInternalEmitForDeterministicMvid->ChangeMvid(deterministicGuid); + + if (FAILED(hr = m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, deterministicTimestamp))) goto exit; + } + if (m_fGeneratePDB) { mdMethodDef entryPoint; @@ -1464,15 +1495,12 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (m_fDeterministic) { - // Get deterministic GUID and timestamp from the computed hash - _ASSERTE(sizeof(GUID) + sizeof(ULONG) <= sizeof(pdbChecksum)); - GUID pdbGuid = *((GUID*)&pdbChecksum); - if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(pdbGuid))) goto exit; - - ULONG timestamp; - memcpy_s(×tamp, sizeof(ULONG), pdbChecksum + sizeof(GUID), sizeof(ULONG)); - m_pPortablePdbWriter->SetTimestamp(timestamp); - if (FAILED(hr = m_pCeeFileGen->SetFileHeaderTimeStamp(m_pCeeFile, timestamp))) goto exit; + // Now that the PDB checksum has been computed, update the GUID and timestamp + _ASSERTE(*(m_pPortablePdbWriter->GetGuid()) == GUID()); + if (FAILED(hr = m_pPortablePdbWriter->ChangePdbStreamGuid(deterministicGuid))) goto exit; + + _ASSERTE(m_pPortablePdbWriter->GetTimestamp() == 0); + m_pPortablePdbWriter->SetTimestamp(deterministicTimestamp); } if (FAILED(hr=CreateDebugDirectory(pdbChecksum))) goto exit; @@ -1480,14 +1508,6 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr=m_pCeeFileGen->SetOutputFileName(m_pCeeFile, pwzOutputFilename))) goto exit; - // Reserve a buffer for the meta-data - DWORD metaDataSize; - if (FAILED(hr=m_pEmitter->GetSaveSize(cssAccurate, &metaDataSize))) goto exit; - BYTE* metaData; - if (FAILED(hr=m_pCeeFileGen->GetSectionBlock(m_pILSection, metaDataSize, sizeof(DWORD), (void**) &metaData))) goto exit; - ULONG metaDataOffset; - if (FAILED(hr=m_pCeeFileGen->GetSectionDataLen(m_pILSection, &metaDataOffset))) goto exit; - metaDataOffset -= metaDataSize; // set managed resource entry, if any if(m_pManifest && m_pManifest->m_dwMResSizeTotal) { @@ -1701,21 +1721,6 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr)) goto exit; } - if (m_fDeterministic) - { - // In deterministic mode, the MVID needs to be stabilized for the metadata scope that was - // created in Assembler::InitMetaData, and it is guaranteed that the IMDInternalEmit for - // that scope was already acquired immediately after that scope was created. - _ASSERTE(m_pInternalEmitForDeterministicMvid != NULL); - _ASSERTE(sizeof(GUID) <= 32); - BYTE hash[32]; - if (FAILED(hr = Sha256Hash(metaData, metaDataSize, hash, sizeof(hash)))) goto exit; - - GUID mvid; - memcpy(&mvid, hash, sizeof(GUID)); - m_pInternalEmitForDeterministicMvid->ChangeMvid(mvid); - } - if(bClock) bClock->cFilegenBegin = GetTickCount(); // actually output the meta-data if (FAILED(hr=m_pCeeFileGen->EmitMetaDataAt(m_pCeeFile, m_pEmitter, m_pILSection, metaDataOffset, metaData, metaDataSize))) goto exit; From 266538d50d7e210ddcc7f96412a878b984292346 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 8 Jan 2025 13:47:02 -0500 Subject: [PATCH 67/69] Fix file renaming in determinism tests on Windows --- src/tests/Common/CLRTest.Jit.targets | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 58860d97fe038..cffc5875f7161 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -378,9 +378,9 @@ for inputAssemblyName in glob.glob("*.dll"): if test_det: try: - os.rename(inputAssemblyName, f'{inputAssemblyName}.base') + os.replace(inputAssemblyName, f'{inputAssemblyName}.base') except: - print("Could not find f'{inputAssemblyName}'") + print(f'Could not find or move {inputAssemblyName}') sys.exit(1) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={inputAssemblyName} {ilasmSwitches} {disassemblyName}' @@ -394,9 +394,9 @@ for inputAssemblyName in glob.glob("*.dll"): sys.exit(1) try: - os.rename(inputAssemblyName, f'{inputAssemblyName}.diff') + os.replace(inputAssemblyName, f'{inputAssemblyName}.diff') except: - print("Could not find f'{inputAssemblyName}'") + print(f'Could not find or move {inputAssemblyName}') sys.exit(1) if not filecmp.cmp(f'{inputAssemblyName}.base', f'{inputAssemblyName}.diff', shallow=False): @@ -425,9 +425,9 @@ for inputAssemblyName in glob.glob("*.dll"): sys.exit(1) try: - os.rename(pdbName, f'{pdbName}.base') + os.replace(pdbName, f'{pdbName}.base') except: - print("Could not find f'{pdbName}'") + print(f'Could not find {pdbName}') sys.exit(1) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' @@ -441,9 +441,9 @@ for inputAssemblyName in glob.glob("*.dll"): sys.exit(1) try: - os.rename(pdbName, f'{pdbName}.diff') + os.replace(pdbName, f'{pdbName}.diff') except: - print("Could not find f{pdbName}") + print(f'Could not find {pdbName}') sys.exit(1) if not filecmp.cmp(f'{pdbName}.base', f'{pdbName}.diff', shallow=False): From c957abfdb71b93af09a6b1ff8ed2916eab18f0b2 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 8 Jan 2025 13:50:02 -0500 Subject: [PATCH 68/69] Error message consistency --- src/tests/Common/CLRTest.Jit.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index cffc5875f7161..8b8ad6d488f65 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -427,7 +427,7 @@ for inputAssemblyName in glob.glob("*.dll"): try: os.replace(pdbName, f'{pdbName}.base') except: - print(f'Could not find {pdbName}') + print(f'Could not find or move {pdbName}') sys.exit(1) ilasm_args = f'{os.environ["CORE_ROOT"]}/ilasm -output={pdbName} {ilasmSwitches} {disassemblyName}' @@ -443,7 +443,7 @@ for inputAssemblyName in glob.glob("*.dll"): try: os.replace(pdbName, f'{pdbName}.diff') except: - print(f'Could not find {pdbName}') + print(f'Could not find or move {pdbName}') sys.exit(1) if not filecmp.cmp(f'{pdbName}.base', f'{pdbName}.diff', shallow=False): From f3da380e577722b5c9e9e99f5f5ee5a51aa700e7 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 8 Jan 2025 14:42:20 -0500 Subject: [PATCH 69/69] Fix build for clang --- src/coreclr/ilasm/writer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/coreclr/ilasm/writer.cpp b/src/coreclr/ilasm/writer.cpp index 8138d7ac6d74c..e075ff9901e47 100644 --- a/src/coreclr/ilasm/writer.cpp +++ b/src/coreclr/ilasm/writer.cpp @@ -1180,6 +1180,8 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) DWORD mresourceSize = 0; BYTE* mresourceData = NULL; WCHAR* wzScopeName = NULL; + GUID deterministicGuid = GUID(); + ULONG deterministicTimestamp = 0; if(bClock) bClock->cMDEmitBegin = GetTickCount(); if(m_fReportProgress) printf("Creating PE file\n"); @@ -1461,9 +1463,6 @@ HRESULT Assembler::CreatePEFile(_In_ __nullterminated WCHAR *pwzOutputFilename) if (FAILED(hr=m_pCeeFileGen->GetSectionDataLen(m_pILSection, &metaDataOffset))) goto exit; metaDataOffset -= metaDataSize; - GUID deterministicGuid = GUID(); - ULONG deterministicTimestamp = 0; - if (m_fDeterministic) { // Get deterministic GUID and timestamp from the computed hash