diff --git a/.config/LocalizationValidationAllowlist.json b/.config/LocalizationValidationAllowlist.json new file mode 100644 index 0000000000..0a3cbfea6b --- /dev/null +++ b/.config/LocalizationValidationAllowlist.json @@ -0,0 +1,110 @@ +{ + "_comment": [ + "These culture/key pairs intentionally match the English source text.", + "Each pair was verified in the internal LCL source as localized (Stat=Loc, Orig=New). Remove an entry when its localized value changes." + ], + "AllowedEnglishValueMatches": { + "Strings.cs.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_Data", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.de.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId" + ], + "Strings.es.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass" + ], + "Strings.fr.resx": [ + "DataCategory_InfoMessage", + "DataCategory_Notification", + "DataCategory_Source", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SqlMisc_NullString" + ], + "Strings.it.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.ja.resx": [ + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId" + ], + "Strings.ko.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.pl.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.pt-BR.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass" + ], + "Strings.ru.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExOriginalClientConnectionId" + ], + "Strings.tr.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.zh-Hans.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.zh-Hant.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ] + } +} diff --git a/.config/PolicheckExclusions.xml b/.config/PolicheckExclusions.xml index a4269513d1..e1129013ba 100644 --- a/.config/PolicheckExclusions.xml +++ b/.config/PolicheckExclusions.xml @@ -1,5 +1,5 @@ SRC/MICROSOFT.DATA.SQLCLIENT/TESTS .YML|.MD|.SQL - NOTICE.TXT|SQLDATAADAPTER.CS + NOTICE.TXT|SQLDATAADAPTER.CS|SQLDATAADAPTER.XML \ No newline at end of file diff --git a/.config/guardian/.gdnbaselines b/.config/guardian/.gdnbaselines new file mode 100644 index 0000000000..e5c5ff89b2 --- /dev/null +++ b/.config/guardian/.gdnbaselines @@ -0,0 +1,787 @@ +{ + "hydrated": false, + "properties": { + "helpUri": "https://eng.ms/docs/microsoft-security/security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/microsoft-guardian/general/baselines" + }, + "version": "1.0.0", + "baselines": { + "default": { + "name": "default", + "createdDate": "2026-07-23 11:29:23Z", + "lastUpdatedDate": "2026-08-28 14:33:24Z" + } + }, + "results": { + "40b23e076c5c65b58c7da0889f0bf58e271b1637f662060bbc7aadd5c62a7126": { + "signature": "40b23e076c5c65b58c7da0889f0bf58e271b1637f662060bbc7aadd5c62a7126", + "alternativeSignatures": [ + "4113c2e383bc4d7206cc94ba3e16dcdaef105762c7e22060d8e7964407e048ab", + "41bc93496e4ace0362f89323cc166b46e7448108052b75f7dd3ae5f4d48b5ad8", + "357246d2ddf96dbab309e392e49d088dc5c15ec9343553876173296f02eef94c" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "6e0dddb2d631181c20dbbf2892aceea3fbb157b3c293a147b70f49fc4a17765b": { + "signature": "6e0dddb2d631181c20dbbf2892aceea3fbb157b3c293a147b70f49fc4a17765b", + "alternativeSignatures": [ + "2a9d1465faa5347c1f77eb856eb21074dc1dfcca4963991fb918fa61c1c29b02", + "9f71983dfef73a0b5fa1ecd3d1f14b3af9067aed39c5eb8619ecd0a0f4ee33f6", + "bd1475a089ab85c698803c80d71e73148edfc9db706bafe3789509ea32f917f2" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "28f157377f5600a0ce77a1a2193653ca71c8de4cca5d24166e1363cfc1f53782": { + "signature": "28f157377f5600a0ce77a1a2193653ca71c8de4cca5d24166e1363cfc1f53782", + "alternativeSignatures": [ + "ff91ec920d3ed908c7fdc8419f4e767216c312a3907419b65e8e11e611be0db6", + "cc5e7e0d3670c5b2738f2b9194d50de2b123bdae5e906729f3a9c7baa980477d", + "8641e063e4aff04c260660c01c71ee18d32bdb7788357eaecc420a64cb279b2d" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "253df4b6cde68fb64c0376d3fa5351a49ccd9136e987129a6686f003303902bb": { + "signature": "253df4b6cde68fb64c0376d3fa5351a49ccd9136e987129a6686f003303902bb", + "alternativeSignatures": [ + "e98757960f13dfe3c2c4ef2acbbdbd782cf26a7635caae25b7fd9e9230f863f6", + "df606344d52fe66ab4fa386e5cc50b6ced2c8d711aa7bc5a14db61f1fad7b70f", + "61239d49bfbe4702066454a51c302633291f61542011ee755420f4f556ede2d6" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "40b7b29173191dd629e8921c99d8718cad7ebf4d6acd83d8719e2e15db3d89a3": { + "signature": "40b7b29173191dd629e8921c99d8718cad7ebf4d6acd83d8719e2e15db3d89a3", + "alternativeSignatures": [ + "870b4398ab01cdfaa4be8605e03eaa7377a5079b79573d3290248f5d5fd6fd6c", + "6b690695e0e41aa7a7c79394fa5d9f6a08e6e06cffc326b7fae4a8883ff42ac6", + "514ed34ad14af5b3739b2804e791b1b63d88be42e658238794f467c4bf2ff140" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "fb0afe300269068323c4e3fab292104aa30201c545156ffee1ccf6109df3bd59": { + "signature": "fb0afe300269068323c4e3fab292104aa30201c545156ffee1ccf6109df3bd59", + "alternativeSignatures": [ + "7bdac882cf7914eb42130101ca921f1e4c5654cc55916daf1c2b9bc51c439dfa", + "6e99b53e08dcbad4533fbd8a101b881fba112f31483c7e9b7b14aaa709f3cee8", + "fc771d2d8a5bd2dfa72695e42516ff6c504e18fd714a1462d911539ea69fe710" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "e3ec17e281386f24ee7650c502b034993dcfd9a5ad19089c9af3edde8321fbb9": { + "signature": "e3ec17e281386f24ee7650c502b034993dcfd9a5ad19089c9af3edde8321fbb9", + "alternativeSignatures": [ + "0910ddd1f1c9f37b764457a39d0dd543f25fce6e5e8694f066d0092e99784b4d", + "ee166a838d6e6b4aa08218eb89b001689bc3ec8fb6343fdfee89834847a294d5", + "7beed57053ec779f46498ff84c41a9400761001ea494237fe32cfab1c829454a" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "0ef75f3e883bfae828f2b15b4a41963e6ea3a61306fc19767586044309bc739f": { + "signature": "0ef75f3e883bfae828f2b15b4a41963e6ea3a61306fc19767586044309bc739f", + "alternativeSignatures": [ + "7e71c00bf08e29508821cc200dcc41556ecc570c205f8833a5a704466e80baa5", + "0d5b851e97bdb0eb8931e6f326718a657f9cd46092eb8a2a2d414be2147a797c", + "e6c0cd6ef2433a42c95a2939cce740019ecda3fcfde64a3a0f21661e6ff27f71" + ], + "target": "src/Microsoft.Data.SqlClient/tests/ManualTests/makepfxcert.ps1", + "line": 145, + "uriBaseId": "file:///D:/a/_work/1/s/", + "memberOf": [ + "default" + ], + "tool": "psscriptanalyzer", + "ruleId": "PSAvoidUsingConvertToSecureStringWithPlainText", + "createdDate": "2026-08-28 12:58:19Z" + }, + "27cf35f7df3f630fab489573ec19318f563e042424ca30625e9fe08407d74bdf": { + "signature": "27cf35f7df3f630fab489573ec19318f563e042424ca30625e9fe08407d74bdf", + "alternativeSignatures": [ + "55e9029f79f883ee7e23a98ee6aad0c3713e02dd67cd7a7158875cdee4c86806", + "a443d3867a75710110ddf675eb639080559e694d4331fd89bc1589b1a60b8777", + "ff040e311fa90fc602c647b539f52148605321c178fc731f30395a8a9ef3f7a9" + ], + "memberOf": [ + "default" + ], + "createdDate": "2026-07-23 11:29:23Z" + }, + "1e0989a7cdd65afb10dd3787a2ed33e9f737e6dd049524edbf76ba1daadf6ef8": { + "signature": "1e0989a7cdd65afb10dd3787a2ed33e9f737e6dd049524edbf76ba1daadf6ef8", + "alternativeSignatures": [ + "47067564034219f2cf40604fcd1beadb34ebe1b960cc75600796c8a0f4565604" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/src/Utils.cs", + "line": 72, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 14:17:20Z" + }, + "e5cd66384b36741191c98844947e6b857140c09b2c352b180664f8b4829c3e4c": { + "signature": "e5cd66384b36741191c98844947e6b857140c09b2c352b180664f8b4829c3e4c", + "alternativeSignatures": [ + "07fc741e29b6f1d01d84d3290b8c53dc7f22036a8313d1f41acdca253aa3e254" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Resources/StringsHelper.cs", + "line": 90, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "281a106076dd3d70aefcd230a90cef542f52488fb3ce164c94b213ededa12da5": { + "signature": "281a106076dd3d70aefcd230a90cef542f52488fb3ce164c94b213ededa12da5", + "alternativeSignatures": [ + "42cf7833cc5d63447162200f8926b57746ad3f6f2bd43318f0e606f022933c3e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/AzureAttestationBasedEnclaveProvider.cs", + "line": 215, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "fcda2db01b63d59f5a4aa73cb780405887f4616f26f9e23dfae52195cab4994e": { + "signature": "fcda2db01b63d59f5a4aa73cb780405887f4616f26f9e23dfae52195cab4994e", + "alternativeSignatures": [ + "9134dead4de902cc02f5f2e7785d84b422f31847f237ba6bcbaeb823e228fd7d" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveProviderBase.cs", + "line": 170, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "4ff2eb551fd17b4d3239b89cf97bfb09507cdfb631cb4787adc3a4f195e8bac7": { + "signature": "4ff2eb551fd17b4d3239b89cf97bfb09507cdfb631cb4787adc3a4f195e8bac7", + "alternativeSignatures": [ + "3b7985cbb5123ba5b91edc3e36a952d1f9d579943820f3c3e870095c7e264cc4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/NoneAttestationEnclaveProvider.cs", + "line": 43, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8f3e9755a800f590583d9e7ea2028c5d23ee2008450af7d55daea17d5fbecfa5": { + "signature": "8f3e9755a800f590583d9e7ea2028c5d23ee2008450af7d55daea17d5fbecfa5", + "alternativeSignatures": [ + "4531f356921a98edacfca7c32eb389b459c02014ea3db2d2d899ce65a87d52ca" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlAeadAes256CbcHmac256EncryptionKey.cs", + "line": 94, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "6ba87c464ec17dc52c0304f5cee224f8d3233507a79211916901a7b9d0d0908c": { + "signature": "6ba87c464ec17dc52c0304f5cee224f8d3233507a79211916901a7b9d0d0908c", + "alternativeSignatures": [ + "fcd1d28e2fe4772861caa303138474dc79869cc77079f79b55eda1612a4c4693" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlAuthenticationProviderManager.cs", + "line": 353, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "ffe242f34f321ccf4d2e727c9812baaf006e4ad122d314e02dd287f388b5b176": { + "signature": "ffe242f34f321ccf4d2e727c9812baaf006e4ad122d314e02dd287f388b5b176", + "alternativeSignatures": [ + "ca2f6e8136bafc65dc12eb6236123ee0877de7a1b48e810c36b7feae643b9654" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs", + "line": 1049, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "b73abd622849352bbe6c5188f106d4a2f9a1c4f650b7d2ba8f383653909a9479": { + "signature": "b73abd622849352bbe6c5188f106d4a2f9a1c4f650b7d2ba8f383653909a9479", + "alternativeSignatures": [ + "636e8fa98391919cfbd7f27792cac5c7806dbd7ec3ab0506b27fbbe52ee9cd8b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientPermission.netfx.cs", + "line": 144, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "cdbeec94358c58eb2960ea291f7f804e5d24e15807bc5b4a5ce259782727cd9a": { + "signature": "cdbeec94358c58eb2960ea291f7f804e5d24e15807bc5b4a5ce259782727cd9a", + "alternativeSignatures": [ + "8fa2f809347c794dbb3659a25cb2ea3a15df3a64a86f5a78f72b39c63f6b8319" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs", + "line": 2336, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "01c8d80a7268d44c7d8e478f887e804d8819cc2382ad1d845a415d97ead39d2e": { + "signature": "01c8d80a7268d44c7d8e478f887e804d8819cc2382ad1d845a415d97ead39d2e", + "alternativeSignatures": [ + "00aae68e847dfaed6240e67d9f0d1f5f64b9a0343c185c69f7dae148ebf2dc35" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionEncryptOption.cs", + "line": 57, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "5ef60ae6fdf13d9fcebe7631af27f7ea23d3f198215a19c8752bf5f8cdee8dc9": { + "signature": "5ef60ae6fdf13d9fcebe7631af27f7ea23d3f198215a19c8752bf5f8cdee8dc9", + "alternativeSignatures": [ + "17d62daf6be556a78b7a342be79f5038d7f1831851722a69398b996cecd57bd8" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionEncryptOption.cs", + "line": 108, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f99754da199aaa7a5e510552f0b4e851029c149dced1d5196eba374b03f0450d": { + "signature": "f99754da199aaa7a5e510552f0b4e851029c149dced1d5196eba374b03f0450d", + "alternativeSignatures": [ + "2852f83af1a5eacc738153cae383e7e216179f3573e9082879dddb48e32a260c" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs", + "line": 1638, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "3dc0ef1e00dd1aed1bd9b6a2e9c06c4f8b24bf368419a2928feaab51252a3b47": { + "signature": "3dc0ef1e00dd1aed1bd9b6a2e9c06c4f8b24bf368419a2928feaab51252a3b47", + "alternativeSignatures": [ + "6a46ad5f8328cb647c28327bb4260335dab6e381ee816f60e371f3f3c4f348b0" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs", + "line": 1640, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d0e489f06819d4a2e78e99be8aedb5d683f8c787cfc07d5c25689bc4e4e3f5f6": { + "signature": "d0e489f06819d4a2e78e99be8aedb5d683f8c787cfc07d5c25689bc4e4e3f5f6", + "alternativeSignatures": [ + "524203b79cc017dc30443712f932710574a9a39d5f7251a4c9354f3cdd21b36b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.Debug.cs", + "line": 59, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "6d1f24e834de7bf17aa69abdd0fe79751a4afcc083253e84b577c813a19f46fb": { + "signature": "6d1f24e834de7bf17aa69abdd0fe79751a4afcc083253e84b577c813a19f46fb", + "alternativeSignatures": [ + "9b805fc1d43486b21ec75d68dd2e4b19d5e7af7c954c546389307bd689a2930b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.Debug.cs", + "line": 59, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d245c0ad54d0229b6ea7a194bd42c40011b734ee5ef92dfce42c9a40096c906d": { + "signature": "d245c0ad54d0229b6ea7a194bd42c40011b734ee5ef92dfce42c9a40096c906d", + "alternativeSignatures": [ + "472b266989720cacdc2666a97234830e954f84346ba13ee028d9f0dbac3d5d82" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs", + "line": 2801, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "68fc925a125fdb6381d42edaaae658cb5c23c5bbef262cbb569215d839e3a9b3": { + "signature": "68fc925a125fdb6381d42edaaae658cb5c23c5bbef262cbb569215d839e3a9b3", + "alternativeSignatures": [ + "1a6475a1a8210fd0e0d4807a4c5d4e1017da257dbfa97b17c8e3382c1684a34a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDependency.cs", + "line": 644, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA2219", + "createdDate": "2026-08-28 13:55:54Z" + }, + "df9958d713606f4b2d800a8f5b33e4839cde7d9e7514f732e0b999f4e7df0cb0": { + "signature": "df9958d713606f4b2d800a8f5b33e4839cde7d9e7514f732e0b999f4e7df0cb0", + "alternativeSignatures": [ + "b44a5e32ae614b1bdda97a1a634dad5a4ab4ede29b463dd3196f309db10e1d15" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDependency.cs", + "line": 1222, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "e318e84cf65f756c454457b437010a21fac2f27f2fa7378c3befc809b97369be": { + "signature": "e318e84cf65f756c454457b437010a21fac2f27f2fa7378c3befc809b97369be", + "alternativeSignatures": [ + "d8118425a1409e87f5983a840ac22cc663a7ab494bb914b312f7b32adb84d5f5" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs", + "line": 1134, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "467568c2128c495889d899ef783ccca652f3b355aa8fe815d4da1b55b6deab95": { + "signature": "467568c2128c495889d899ef783ccca652f3b355aa8fe815d4da1b55b6deab95", + "alternativeSignatures": [ + "8962958e2e5e468cc56038a04d02476e352bf6f1d48104f51c13990d364b0c64" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlException.cs", + "line": 164, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "727c936839845e9a68ff00fbc69abb5a132769506d3cfe87692612a2addac855": { + "signature": "727c936839845e9a68ff00fbc69abb5a132769506d3cfe87692612a2addac855", + "alternativeSignatures": [ + "2450ed8367a7fac65b41524a145bf097f2dee5794752124fefa3a635057946bd" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs", + "line": 114, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8c6b44ec1f44fbe00469a48beb3f9eee61c8e31e237c53ba372a1abfc5ea481f": { + "signature": "8c6b44ec1f44fbe00469a48beb3f9eee61c8e31e237c53ba372a1abfc5ea481f", + "alternativeSignatures": [ + "7a3c05145c302720ca6feadabcacbb11303442b2289fb06796e21b713fd63717" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs", + "line": 572, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8a5592e024f45a9bdce3315e108cccfafe99b184dbcb4e50d59d783fa3db3942": { + "signature": "8a5592e024f45a9bdce3315e108cccfafe99b184dbcb4e50d59d783fa3db3942", + "alternativeSignatures": [ + "929389afe5818dcc5113299f0e0263eca26ef989785742ac7e46b088d5cae598" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs", + "line": 2364, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "770e1d8ef5a06c9b4b552dc9c8bf000f572d633bf11ded7e749b3ac9c07d208b": { + "signature": "770e1d8ef5a06c9b4b552dc9c8bf000f572d633bf11ded7e749b3ac9c07d208b", + "alternativeSignatures": [ + "4cd0e3d7087eaa0eaf05bec58d32fd71b660033f7bf3984392998d0b9946fb47" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlSecurityUtility.cs", + "line": 389, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8502b61241cdbfac98f7c84b8c16d4d4ec0ea2185018d9fce710e2bac445e8b0": { + "signature": "8502b61241cdbfac98f7c84b8c16d4d4ec0ea2185018d9fce710e2bac445e8b0", + "alternativeSignatures": [ + "afed061c02d7b602c516f09952197b58c6c1cdfa0e07ee3625af48287869b2c5" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs", + "line": 1749, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "a83b0d426733a42a34b4569643e60513b598035d6807fa8cda6f4a3501084929": { + "signature": "a83b0d426733a42a34b4569643e60513b598035d6807fa8cda6f4a3501084929", + "alternativeSignatures": [ + "3614422f0f09ab4fff1a0195a4f40acd53bf55e4781153ced9d736a801fc6aa4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs", + "line": 1875, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d36be8dd5189d09c4b5db9176bda7628839f0ba608f31ba449f1203ca9c30c09": { + "signature": "d36be8dd5189d09c4b5db9176bda7628839f0ba608f31ba449f1203ca9c30c09", + "alternativeSignatures": [ + "fcf7dc6efdfecd535087f2361d43cfa599cbe83b52717beb36c052dadbbc5a5d" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs", + "line": 2309, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "b19ca78715cc12a2051b38a495f41c060259e46d893be38e6dba447e8a87cc02": { + "signature": "b19ca78715cc12a2051b38a495f41c060259e46d893be38e6dba447e8a87cc02", + "alternativeSignatures": [ + "efc4d3f86b80b0d8392e7fa74b15ca51411078ed8f249b0f30911036530bc0b4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs", + "line": 3918, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "119b9514aeb84d5e34b5d7b1520378d4a2d017902d7d1bd572404f1b3186c59c": { + "signature": "119b9514aeb84d5e34b5d7b1520378d4a2d017902d7d1bd572404f1b3186c59c", + "alternativeSignatures": [ + "57389b1211c68b1c8ed21fb3439d1dd9bd8a8e25677ed63b1db38ff7bd3b0c6b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs", + "line": 4403, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "533d9dc25ef8183859aeed955b720d435e3035702b3b7c10d12204170815ec04": { + "signature": "533d9dc25ef8183859aeed955b720d435e3035702b3b7c10d12204170815ec04", + "alternativeSignatures": [ + "55c532f39c15ec069c540c661f1ccb7371ee8c03f74fad4515d9cd94db8bcca1" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObjectNative.cs", + "line": 100, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "90dd8eeec55b68eaf94f971d1af06723d22efc0f7cb5fd15870765e6e6602ff9": { + "signature": "90dd8eeec55b68eaf94f971d1af06723d22efc0f7cb5fd15870765e6e6602ff9", + "alternativeSignatures": [ + "ac02e05713e85c6086fc30c51a7083a159329f0a222c9d0be513e6da9fda0300" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/VirtualSecureModeEnclaveProvider.cs", + "line": 84, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f1c711a9df14d19ac6682356d6648111a1e3c2ae051812e8f04a7466df045aa4": { + "signature": "f1c711a9df14d19ac6682356d6648111a1e3c2ae051812e8f04a7466df045aa4", + "alternativeSignatures": [ + "1e752a5b4246f05f58da2adde778eb3eb46238ad51a6a2013d3fd9d80aaf5422" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/VirtualSecureModeEnclaveProviderBase.cs", + "line": 488, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d159ab04bf10a650f1f43064602246acfa671e4f1f7ba07faf5620ab13043926": { + "signature": "d159ab04bf10a650f1f43064602246acfa671e4f1f7ba07faf5620ab13043926", + "alternativeSignatures": [ + "74e1a9e9972040959e4176eb8dc981d8394545eaf86f30e059f9814f545bea67" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionString.netfx.cs", + "line": 374, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "761ae24086cbfbeb9667e20ab45b174613bd156d0dc7bde56a37571db11cd779": { + "signature": "761ae24086cbfbeb9667e20ab45b174613bd156d0dc7bde56a37571db11cd779", + "alternativeSignatures": [ + "1793d6a0e1d5ee495ced5ce18af4bb1c6f1635ce89d6dfe9d840b84673ccbe7a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs", + "line": 4130, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "92759af0d8bc9a56339a09bfb5ae1e0adbe2c0aa7697911d0f14941ba65a1e67": { + "signature": "92759af0d8bc9a56339a09bfb5ae1e0adbe2c0aa7697911d0f14941ba65a1e67", + "alternativeSignatures": [ + "f20cbad7a6ed702f47f5d26f6e56844ce9f1e5a7d03152361b1818d4aef03058" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolAuthenticationContextKey.cs", + "line": 83, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8409f840dce0e042b55250ce12045afaabc1e2e48f443e43ec0e493517e3e38d": { + "signature": "8409f840dce0e042b55250ce12045afaabc1e2e48f443e43ec0e493517e3e38d", + "alternativeSignatures": [ + "b607bb6c9aded6c9d9d1da95e9f49b9a17ec42880fe59156cfca1d2f088fec35" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniCommon.netcore.cs", + "line": 148, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "1152991aab5d089b6b086e389b075de228b319c82a577dae0afe850956202b12": { + "signature": "1152991aab5d089b6b086e389b075de228b319c82a577dae0afe850956202b12", + "alternativeSignatures": [ + "95a2b83edbb49524b5834cba58e1c1670ad5397cca2bc8ed4912f02c7b885acd" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs", + "line": 150, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "66052c0cd9b0dade1d5d86703650fa9c8d46f7977b60148aea67f371393e32b3": { + "signature": "66052c0cd9b0dade1d5d86703650fa9c8d46f7977b60148aea67f371393e32b3", + "alternativeSignatures": [ + "d67c06a92599b72202abeed718e385e4621947195fc3affc730a1066b55f5cb6" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs", + "line": 731, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f236b2c674fc8574908a9b254e10d9a78f4df86278d9cfe52f5b4d072689829b": { + "signature": "f236b2c674fc8574908a9b254e10d9a78f4df86278d9cfe52f5b4d072689829b", + "alternativeSignatures": [ + "fabdb2e276df6d043b029af9a40c831c8f9f7131dec81706dd664cd5d913f32e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniTcpHandle.netcore.cs", + "line": 658, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "424ec1510b6c1352e0731f19bd7a6c03f1da3bab869fa520046b0080cb7b5d70": { + "signature": "424ec1510b6c1352e0731f19bd7a6c03f1da3bab869fa520046b0080cb7b5d70", + "alternativeSignatures": [ + "4dac74c5b9483f0a4b1f7e76f91002aa485fd1b83b19d5823169b0c15eaace8b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SsrpClient.netcore.cs", + "line": 80, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "76482116b8cae39bdad9b92782ab216ff958f6578405f93879733326a4283bfe": { + "signature": "76482116b8cae39bdad9b92782ab216ff958f6578405f93879733326a4283bfe", + "alternativeSignatures": [ + "a815e32ff0ad176d42719f84cb015df61c89e9539fecbf58da169993e789b682" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs", + "line": 308, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "dd455bea13e4bd70696d1b0f4c890ca1ea42e18643bbe6b572989d84ac188f06": { + "signature": "dd455bea13e4bd70696d1b0f4c890ca1ea42e18643bbe6b572989d84ac188f06", + "alternativeSignatures": [ + "0444dab1ebebce0b0e0bfb453d8513e9e8108aad22055bd038c873a8a8bc1f0e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs", + "line": 629, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:36:06Z" + }, + "09526af76b9a6ad21a4841ea50bb51e72925789c5f0f733650ac946cc44060a9": { + "signature": "09526af76b9a6ad21a4841ea50bb51e72925789c5f0f733650ac946cc44060a9", + "alternativeSignatures": [ + "2a3fb00242d533ee3253b91c42ba7882b9c35ada1635e3470c481c6b43baa0e6" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/SqlUserDefinedAggregateAttribute.netstandard.cs", + "line": 61, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "6fec84f5f9e3087c484965319896c91b033557b8bcef7761b0ae74e1baf0ad7d": { + "signature": "6fec84f5f9e3087c484965319896c91b033557b8bcef7761b0ae74e1baf0ad7d", + "alternativeSignatures": [ + "efa9c07656c41d1d2367a89a7146fda28f3b584e2654693b4954d40bfafecb8f" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/SqlUserDefinedTypeAttribute.netstandard.cs", + "line": 73, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "1de5740b9e122c2d8bda4bfe66c94764a6192376f753a7234bac91e1fc28e5f6": { + "signature": "1de5740b9e122c2d8bda4bfe66c94764a6192376f753a7234bac91e1fc28e5f6", + "alternativeSignatures": [ + "14c7177139597c2ab94cf632b76bc03c4f2c252691b30a7ba69957e047b400c4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/StringsHelper.netstandard.cs", + "line": 130, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "0ca59be802da38e82370950b24300deae99834aa9a3cee38e064d1f17c5942a9": { + "signature": "0ca59be802da38e82370950b24300deae99834aa9a3cee38e064d1f17c5942a9", + "alternativeSignatures": [ + "26425ecc80bd3289865ce79ae6471c4ab21cde25acc1cf2067717fbf6d600a6a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.Internal/Logging/src/SqlClientEventSource.cs", + "line": 1995, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:09:32Z" + } + } +} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c424e37834..db5f1c5da5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -16,7 +16,7 @@ "ms-mssql.mssql" ], "settings": { - "dotnet.defaultSolution": "src/Microsoft.Data.SqlClient.sln" + "dotnet.defaultSolution": "src/Microsoft.Data.SqlClient.slnx" } } }, diff --git a/.devcontainer/setup-sqlserver.sh b/.devcontainer/setup-sqlserver.sh index 6b4ee480ed..fc962c3053 100755 --- a/.devcontainer/setup-sqlserver.sh +++ b/.devcontainer/setup-sqlserver.sh @@ -29,11 +29,11 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" # Write test config file so the test suite can find the connection string. CONFIG_DIR="${REPO_ROOT}/src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities" -CONFIG_DEFAULT_FILE="${CONFIG_DIR}/config.default.json" -CONFIG_FILE="${CONFIG_DIR}/config.json" +CONFIG_DEFAULT_FILE="${CONFIG_DIR}/config.default.jsonc" +CONFIG_FILE="${CONFIG_DIR}/config.jsonc" echo "Writing test config to ${CONFIG_FILE} (based on ${CONFIG_DEFAULT_FILE})..." TCP_CONN_STR="Data Source=tcp:${SQL_HOST},${SQL_PORT};Database=Northwind;User Id=sa;Password=${SA_PASSWORD};Encrypt=false;TrustServerCertificate=true" -# config.default.json contains JS-style comments (// ...) which are not valid JSON. +# config.default.jsonc contains JS-style comments (// ...) which are not valid JSON. # Strip single-line comments before feeding to jq. sed 's|//.*||' "${CONFIG_DEFAULT_FILE}" \ | jq --arg cs "${TCP_CONN_STR}" \ diff --git a/.editorconfig b/.editorconfig index 819a41fe1e..1864df162c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -197,4 +197,4 @@ dotnet_diagnostic.CA1416.severity = silent dotnet_code_quality.CA2100.excluded_type_names_with_derived_types = Microsoft.Data.SqlClient.ManualTesting.Tests.* dotnet_diagnostic.xUnit1031.severity=none -dotnet_diagnostic.xUnit1030.severity=none +dotnet_diagnostic.xUnit1030.severity=none \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index 1ff0c42304..ab4f136a23 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,63 +1,5 @@ -############################################################################### -# Set default behavior to automatically normalize line endings. -############################################################################### +# Normalize line endings * text=auto -############################################################################### -# Set default behavior for command prompt diff. -# -# This is need for earlier builds of msysgit that does not have it on by -# default for csharp files. -# Note: This is only used by command line -############################################################################### -#*.cs diff=csharp - -############################################################################### -# Set the merge driver for project and solution files -# -# Merging from the command prompt will add diff markers to the files if there -# are conflicts (Merging from VS is not affected by the settings below, in VS -# the diff markers are never inserted). Diff markers may cause the following -# file extensions to fail to load in VS. An alternative would be to treat -# these files as binary and thus will always conflict and require user -# intervention with every merge. To do so, just uncomment the entries below -############################################################################### -#*.sln merge=binary -#*.csproj merge=binary -#*.vbproj merge=binary -#*.vcxproj merge=binary -#*.vcproj merge=binary -#*.dbproj merge=binary -#*.fsproj merge=binary -#*.lsproj merge=binary -#*.wixproj merge=binary -#*.modelproj merge=binary -#*.sqlproj merge=binary -#*.wwaproj merge=binary - -############################################################################### -# behavior for image files -# -# image files are treated as binary by default. -############################################################################### -#*.jpg binary -#*.png binary -#*.gif binary - -############################################################################### -# diff behavior for common document formats -# -# Convert binary document formats to text before diffing them. This feature -# is only available from the command line. Turn it on by uncommenting the -# entries below. -############################################################################### -#*.doc diff=astextplain -#*.DOC diff=astextplain -#*.docx diff=astextplain -#*.DOCX diff=astextplain -#*.dot diff=astextplain -#*.DOT diff=astextplain -#*.pdf diff=astextplain -#*.PDF diff=astextplain -#*.rtf diff=astextplain -#*.RTF diff=astextplain +# Treat workflow lock files as generated +.github/workflows/*.lock.yml linguist-generated=true diff --git a/.github/agents/agentic-workflows.md b/.github/agents/agentic-workflows.md new file mode 100644 index 0000000000..b824e60580 --- /dev/null +++ b/.github/agents/agentic-workflows.md @@ -0,0 +1,226 @@ +--- +name: Agentic Workflows +description: GitHub Agentic Workflows (gh-aw) - Create, debug, and upgrade AI-powered workflows with intelligent prompt routing. +disable-model-invocation: true +--- + +# GitHub Agentic Workflows Agent + +This agent helps you work with **GitHub Agentic Workflows (gh-aw)**, a CLI extension for creating AI-powered workflows in natural language using markdown files. + +## What This Agent Does + +This is a **dispatcher agent** that routes your request to the appropriate specialized prompt based on your task: + +- **Creating new workflows**: Routes to `create` prompt +- **Updating existing workflows**: Routes to `update` prompt +- **Debugging workflows**: Routes to `debug` prompt +- **Upgrading workflows**: Routes to `upgrade-agentic-workflows` prompt +- **Creating report-generating workflows**: Routes to `report` prompt — consult this whenever the workflow posts status updates, audits, analyses, or any structured output as issues, discussions, or comments +- **Creating shared components**: Routes to `create-shared-agentic-workflow` prompt +- **Fixing Dependabot PRs**: Routes to `dependabot` prompt — use this when Dependabot opens PRs that modify generated manifest files (`.github/workflows/package.json`, `.github/workflows/requirements.txt`, `.github/workflows/go.mod`). Never merge those PRs directly; instead update the source `.md` files and rerun `gh aw compile --dependabot` to bundle all fixes +- **Analyzing test coverage**: Routes to `test-coverage` prompt — consult this whenever the workflow reads, analyzes, or reports on test coverage data from PRs or CI runs +- **Rendering ASCII charts in markdown**: Routes to `asciicharts` guide — consult this whenever the workflow needs compact charts that render reliably in GitHub issues, comments, or discussions +- **CLI commands and triggering workflows**: Routes to `cli-commands` guide — consult this whenever the user asks how to run, compile, debug, or manage workflows from the command line, or when they need the MCP tool equivalent of a `gh aw` command +- **Reducing token consumption / cost optimization**: Routes to `token-optimization` guide — consult this whenever the user asks how to reduce token usage, lower costs, speed up workflows, or measure the impact of prompt changes with experiments +- **Choosing workflow architectures and design patterns**: Routes to `patterns` guide — consult this whenever the user asks for strategy, architecture, operating models, or pattern selection for agentic workflows + +Workflows may optionally include: + +- **Project tracking / monitoring** (GitHub Projects updates, status reporting) +- **Orchestration / coordination** (one workflow assigning agents or dispatching and coordinating other workflows) + +## Files This Applies To + +- Workflow files: `.github/workflows/*.md` and `.github/workflows/**/*.md` +- Workflow lock files: `.github/workflows/*.lock.yml` +- Shared components: `.github/workflows/shared/*.md` +- Configuration: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/github-agentic-workflows.md` + +## Problems This Solves + +- **Workflow Creation**: Design secure, validated agentic workflows with proper triggers, tools, and permissions +- **Workflow Debugging**: Analyze logs, identify missing tools, investigate failures, and fix configuration issues +- **Version Upgrades**: Migrate workflows to new gh-aw versions, apply codemods, fix breaking changes +- **Component Design**: Create reusable shared workflow components that wrap MCP servers + +## How to Use + +When you interact with this agent, it will: + +1. **Understand your intent** - Determine what kind of task you're trying to accomplish +2. **Route to the right prompt** - Load the specialized prompt file for your task +3. **Execute the task** - Follow the detailed instructions in the loaded prompt + +## Available Prompts + +> **Note**: The prompt and reference files listed below are located in the [`github/gh-aw`](https://github.com/github/gh-aw) repository and are **not available locally** in this repository. Load them from their public URLs. + +### Create New Workflow +**Load when**: User wants to create a new workflow from scratch, add automation, or design a workflow that doesn't exist yet + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/create-agentic-workflow.md` + +**Use cases**: +- "Create a workflow that triages issues" +- "I need a workflow to label pull requests" +- "Design a weekly research automation" + +### Update Existing Workflow +**Load when**: User wants to modify, improve, or refactor an existing workflow + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/update-agentic-workflow.md` + +**Use cases**: +- "Add web-fetch tool to the issue-classifier workflow" +- "Update the PR reviewer to use discussions instead of issues" +- "Improve the prompt for the weekly-research workflow" + +### Debug Workflow +**Load when**: User needs to investigate, audit, debug, or understand a workflow, troubleshoot issues, analyze logs, or fix errors + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/debug-agentic-workflow.md` + +**Use cases**: +- "Why is this workflow failing?" +- "Analyze the logs for workflow X" +- "Investigate missing tool calls in run #12345" + +### Upgrade Agentic Workflows +**Load when**: User wants to upgrade workflows to a new gh-aw version or fix deprecations + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/upgrade-agentic-workflows.md` + +**Use cases**: +- "Upgrade all workflows to the latest version" +- "Fix deprecated fields in workflows" +- "Apply breaking changes from the new release" + +### Create a Report-Generating Workflow +**Load when**: The workflow being created or updated produces reports — recurring status updates, audit summaries, analyses, or any structured output posted as a GitHub issue, discussion, or comment + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/report.md` + +**Use cases**: +- "Create a weekly CI health report" +- "Post a daily security audit to Discussions" +- "Add a status update comment to open PRs" + +### Create Shared Agentic Workflow +**Load when**: User wants to create a reusable workflow component or wrap an MCP server + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/create-shared-agentic-workflow.md` + +**Use cases**: +- "Create a shared component for Notion integration" +- "Wrap the Slack MCP server as a reusable component" +- "Design a shared workflow for database queries" + +### Fix Dependabot PRs +**Load when**: User needs to close or fix open Dependabot PRs that update dependencies in generated manifest files (`.github/workflows/package.json`, `.github/workflows/requirements.txt`, `.github/workflows/go.mod`) + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/dependabot.md` + +**Use cases**: +- "Fix the open Dependabot PRs for npm dependencies" +- "Bundle and close the Dependabot PRs for workflow dependencies" +- "Update @playwright/test to fix the Dependabot PR" + +### Analyze Test Coverage +**Load when**: The workflow reads, analyzes, or reports test coverage — whether triggered by a PR, a schedule, or a slash command. Always consult this prompt before designing the coverage data strategy. + +**Prompt file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/test-coverage.md` + +**Use cases**: +- "Create a workflow that comments coverage on PRs" +- "Analyze coverage trends over time" +- "Add a coverage gate that blocks PRs below a threshold" + +### CLI Commands Reference +**Load when**: The user asks how to run, compile, debug, or manage workflows from the command line; needs the MCP tool equivalent of a `gh aw` command; or is in a restricted environment (e.g., Copilot Cloud) without direct CLI access. + +**Reference file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/cli-commands.md` + +**Use cases**: +- "How do I trigger workflow X on the main branch?" +- "What's the MCP equivalent of `gh aw logs`?" +- "I'm in Copilot Cloud — how do I compile a workflow?" +- "Show me all available gh aw commands" + +### Token Consumption Optimization +**Load when**: The user asks how to reduce token usage, lower workflow costs, make a workflow faster or cheaper, or measure the impact of prompt or configuration changes. + +**Reference file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/token-optimization.md` + +**Use cases**: +- "How do I reduce the token cost of this workflow?" +- "My workflow is too expensive — how do I optimize it?" +- "How do I compare token usage between two runs?" +- "Should I use gh-proxy or the MCP server?" +- "How do I use sub-agents to reduce costs?" +- "How do I measure the impact of a prompt change?" + +### Workflow Pattern Selection +**Load when**: The user asks for architecture, strategy, operating model selection, or pattern recommendations for building agentic workflows. + +**Reference file**: `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/patterns.md` + +**Use cases**: +- "Which pattern should I use for multi-repo rollout?" +- "How should I structure this workflow architecture?" +- "What pattern fits slash-command triage?" +- "Should this be DispatchOps or DailyOps?" + +## Instructions + +When a user interacts with you: + +1. **Identify the task type** from the user's request +2. **Load the appropriate prompt** from the URLs listed above +3. **Follow the loaded prompt's instructions** exactly +4. **If uncertain**, ask clarifying questions to determine the right prompt + +## Quick Reference + +```bash +# Initialize repository for agentic workflows +gh aw init + +# Generate the lock file for a workflow +gh aw compile [workflow-name] + +# Trigger a workflow on demand (preferred over gh workflow run) +gh aw run # interactive input collection +gh aw run --ref main # run on a specific branch + +# Debug workflow runs +gh aw logs [workflow-name] +gh aw audit + +# Upgrade workflows +gh aw fix --write +gh aw compile --validate +``` + +## Key Features of gh-aw + +- **Natural Language Workflows**: Write workflows in markdown with YAML frontmatter +- **AI Engine Support**: Copilot, Claude, Codex, or custom engines +- **MCP Server Integration**: Connect to Model Context Protocol servers for tools +- **Safe Outputs**: Structured communication between AI and GitHub API +- **Strict Mode**: Security-first validation and sandboxing +- **Shared Components**: Reusable workflow building blocks +- **Repo Memory**: Persistent git-backed storage for agents +- **Sandboxed Execution**: All workflows run in the Agent Workflow Firewall (AWF) sandbox, enabling full `bash` and `edit` tools by default + +## Important Notes + +- Always reference the instructions file at `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/github-agentic-workflows.md` for complete documentation +- Use the MCP tool `agentic-workflows` when running in GitHub Copilot Cloud +- Workflows must be compiled to `.lock.yml` files before running in GitHub Actions +- **Bash tools are enabled by default** - Don't restrict bash commands unnecessarily since workflows are sandboxed by the AWF +- Follow security best practices: minimal permissions, explicit network access, no template injection +- **Network configuration**: Use ecosystem identifiers (`node`, `python`, `go`, etc.) or explicit FQDNs in `network.allowed`. Bare shorthands like `npm` or `pypi` are **not** valid. See `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/network.md` for the full list of valid ecosystem identifiers and domain patterns. +- **Single-file output**: When creating a workflow, produce exactly **one** workflow `.md` file. Do not create separate documentation files (architecture docs, runbooks, usage guides, etc.). If documentation is needed, add a brief `## Usage` section inside the workflow file itself. +- **Triggering runs**: Always use `gh aw run ` to trigger a workflow on demand — not `gh workflow run .lock.yml`. `gh aw run` handles workflow resolution by short name, input parsing and validation, and correct run-tracking for agentic workflows. Use `--ref ` to run on a specific branch. +- **CLI commands reference**: For a complete guide on all `gh aw` commands and their MCP tool equivalents (for restricted environments), see `https://raw.githubusercontent.com/github/gh-aw/main/.github/aw/cli-commands.md` diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 0000000000..cdcb39f6a2 --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v9.0.0": { + "repo": "actions/github-script", + "version": "v9.0.0", + "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" + }, + "github/gh-aw-actions/setup@v0.88.2": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.88.2", + "sha": "9271a1804551c0dc4fb0085a97979950aa2f8489" + } + } +} diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 272a0c2164..c0bbc25457 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -11,17 +11,18 @@ ## 📚 Project Overview This project is a .NET data provider for SQL Server, enabling .NET applications to interact with SQL Server databases. It supports various features like connection pooling, transaction management, and asynchronous operations. -The project builds from a **single unified project** at `src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj` that multi-targets `net462`, `net8.0`, and `net9.0`. The legacy `netfx/` and `netcore/` directories are being phased out — only their `ref/` folders (which define the public API surface) remain active. +The project builds from a **single unified project** at `src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj`. It targets `net8.0` and `net9.0` on all supported hosts, and adds `net462` only when building on Windows. The legacy `netfx/` and `netcore/` directories are being phased out — only their `ref/` folders (which define the public API surface) remain active. The project includes: - **Public APIs**: Defined in `netcore/ref/` and `netfx/ref/` directories. - **Implementations**: All source code in `src/Microsoft.Data.SqlClient/src/`. - **Tests**: Located in the `tests/` directory, covering unit and integration tests. - - **Unit Tests**: Located in `tests/UnitTests/` directory, which includes tests for individual components and methods. - - **Functional Tests**: Located in `tests/FunctionalTests/` directory, which includes tests for various features and functionalities that can be run without a SQL Server instance. - - **Manual Tests**: Located in `tests/ManualTests/` directory, which includes tests that require a SQL Server instance to run. + - **Unit Tests**: Located in `src/Microsoft.Data.SqlClient/tests/UnitTests/`. + - **Functional Tests**: Located in `src/Microsoft.Data.SqlClient/tests/FunctionalTests/`. + - **Manual Tests**: Located in `src/Microsoft.Data.SqlClient/tests/ManualTests/`. + - **Performance/Stress Tests**: Located in `src/Microsoft.Data.SqlClient/tests/PerformanceTests/` and `src/Microsoft.Data.SqlClient/tests/StressTests/`. - **Documentation**: Found in the `doc/` directory, including API documentation, usage examples. - **Policies**: Contribution guidelines, coding standards, and review policies in the `policy/` directory. -- **Building**: The project uses MSBuild for building and testing, with configurations and targets defined in the `build.proj` file, whereas instructions are provided in the `BUILDGUIDE.md` file. +- **Building**: The repo uses `build.proj` for orchestrated build/test/pack workflows, `src/Microsoft.Data.SqlClient.slnx` for solution-centric development tooling, and Azure DevOps YAML under `eng/pipelines/` plus `eng/pipelines/onebranch/` for CI and official release flows. See `BUILDGUIDE.md` for local build details. - **CI/CD**: ADO Pipelines for CI/CD and Pull request validation are defined in the `eng/` directory, ensuring code quality and automated testing. ## 📦 Products @@ -33,7 +34,7 @@ This project includes several key products and libraries that facilitate SQL Ser ## 🛠️ Key Features - **Connectivity to SQL Server**: Provides robust and secure connections to SQL Server databases, using various authentication methods, such as Windows Authentication, SQL Server Authentication, and Entra ID authentication, e.g. `ActiveDirectoryIntegrated`, `ActiveDirectoryPassword`, `ActiveDirectoryServicePrincipal`,`ActiveDirectoryInteractive`, `ActiveDirectoryDefault`, and `ActiveDirectoryManagedIdentity`. - **Connection Resiliency**: Implements connection resiliency features to handle transient faults and network issues, ensuring reliable database connectivity. -- **TLS Encryption**: Supports secure connections using TLS protocols to encrypt data in transit. Supports TLS 1.2 and higher, ensuring secure communication with SQL Server. Supported encryption modes are: +- **TLS Encryption**: Supports secure connections using TLS protocols to encrypt data in transit. Supports TLS 1.2 and higher, ensuring secure communication with SQL Server. Supported encryption modes are: - **Optional**: Encryption is used if available, but not required. - **Mandatory**: Encryption is mandatory for the connection. - **Strict**: Enforces strict TLS requirements, ensuring only secure connections are established. @@ -49,6 +50,7 @@ This project includes several key products and libraries that facilitate SQL Ser - **Data Encryption**: Supports data encryption for secure data transmission. - **Logging and Diagnostics**: Provides event source tracing diagnostic capabilities for troubleshooting. - **Failover Support**: Handles automatic failover scenarios for high availability. + - Compatibility switch: `Switch.Microsoft.Data.SqlClient.UseLegacyFailoverAlternationOnLoginSqlErrors` (default `false`) can restore legacy alternation behavior in `LoginWithFailover` for login-phase SQL errors. - **Cross-Platform Support**: Compatible with both .NET Framework and .NET Core, allowing applications to run on Windows, Linux, and macOS. - **Column Encryption AKV Provider**: Supports Azure Key Vault (AKV) provider for acquiring keys from Azure Key Vault to be used for encryption and decryption. @@ -122,13 +124,17 @@ When a new issue is created, follow these steps: - Ensure the PR passes all CI checks before merging. ### ✅ Closing Issues -- Add a comment summarizing the fix and referencing the PR +- Add a comment summarizing the fix and referencing the PR ### ⚙️ Automating Workflows - Auto-label PRs based on folder paths (e.g., changes in `src/Microsoft.Data.SqlClient/src/` → `Area\SqlClient`, changes in `tests/` → `Area\Testing`) and whether they add new public APIs or introduce a breaking change. - Suggest release note entries for fixes by updating files under `release-notes/` or by using the `release-notes` prompt (instead of editing `CHANGELOG.md` directly). - Tag reviewers based on `CODEOWNERS` file +## 🌿 Branch Naming +- All branches created by AI agents **must** use the `dev/automation/` prefix (e.g. `dev/automation/fix-connection-timeout`). +- Do **not** create branches directly under `main`, `dev/`, or any other top-level prefix. + ## 🧠 Contextual Awareness - All source code is in `src/Microsoft.Data.SqlClient/src/`. Do NOT add code to legacy `netfx/src/` or `netcore/src/` directories. - Only `ref/` folders in `netcore/ref/` and `netfx/ref/` remain active for defining the public API surface. @@ -145,6 +151,14 @@ When a new issue is created, follow these steps: - Do not modify `CHANGELOG.md` unless executing a release workflow (see `release-notes` prompt). - Do not close issues without a fix or without providing a clear reason. +## Terminal Execution Safety +- Treat any non-zero shell exit code as a failed step that requires correction before proceeding. +- If a bash process exits, do not wait for more output from that process; rerun the command in a fresh terminal session. +- Validate that expected command output was produced before using it as evidence for conclusions. +- When terminal execution fails, surface the failure immediately and retry with a corrected command. +- Avoid `set -e` in this automation workflow; use focused commands and verify each result explicitly so shell exits are observable and attributable. +- Prefer short, single-purpose terminal commands over long chained scripts when debugging or gathering state. + ## 📝 Notes - Update policies and guidelines in the `policy/` directory as needed based on trending practices and team feedback. - Regularly review and update the `doc/` directory to ensure it reflects the current state of the project. diff --git a/.github/instructions/3rd-party-package-versions.instructions.md b/.github/instructions/3rd-party-package-versions.instructions.md new file mode 100644 index 0000000000..1de7df26bf --- /dev/null +++ b/.github/instructions/3rd-party-package-versions.instructions.md @@ -0,0 +1,208 @@ +--- +applyTo: "**/Directory.Packages.props,**/*.csproj,**/Directory.Build.props,**/*.nuspec" +--- +# Choosing Third-Party Package Dependency Versions + +Guidance for choosing versions of **external (third-party) NuGet package dependencies** in multi-targeted projects (e.g. `net462;net8.0;net9.0`). + +> **Scope:** This document covers dependencies consumed from NuGet — packages the SqlClient repo does NOT own. For versioning of SqlClient's own inter-sibling packages (Logging, Abstractions, SqlClient, Azure, AKV Provider, SqlServer.Server), see `sqlclient-package-versions.instructions.md`. + +## Rule +For runtime-aligned packages, **the package major must match the target runtime major**: 8.x on `net8.0`, 9.x on `net9.0`, 10.x on `net10.0`, and so on. TFMs that aren't tied to a specific runtime major (`net462`, `netstandard2.0`) get the major of the floor LTS. Other categories are versioned as described below. + +Split package references into three categories: + +### 1. Runtime-aligned packages — **version per TFM, matching the runtime band** + +Packages whose major version ships with (or is tightly coupled to) a specific .NET runtime: + +- `Microsoft.Extensions.*` (Logging, DependencyInjection, Configuration, Hosting, Options, Caching, Http, Primitives, ...) +- `Microsoft.AspNetCore.*` +- `Microsoft.EntityFrameworkCore.*` +- `System.Text.Json`, `System.Memory`, `System.IO.Pipelines`, `System.Formats.Asn1`, `System.Security.Cryptography.Pkcs` +- `Microsoft.Bcl.*` + +Use the major version that matches the TFM. For TFMs without a corresponding runtime major (`net462`, `netstandard2.0`, etc.), use the major of the **lowest supported modern TFM** — typically the floor LTS (e.g. `8.x` while net8 is supported). This keeps the legacy targets on a long-lived, well-patched band and avoids dragging in transitive deps from a newer major: + +```xml + + + + + + + + + + +``` + +When the floor LTS drops out of support, bump the default block to the new floor LTS major and drop any conditional block that becomes redundant. + +### 2. Independent packages — **single version across all TFMs** + +Packages whose versioning is decoupled from the .NET runtime: + +- `Newtonsoft.Json`, `Polly.*`, `Serilog.*` +- `Azure.*`, `Microsoft.Identity.*`, `Microsoft.IdentityModel.*` +- `Microsoft.Data.SqlClient`, `Dapper`, `StackExchange.Redis` +- Most third-party packages + +Reference one (latest stable) version unconditionally: + +```xml + + +``` + +### 3. Polyfills — **conditional presence, single version** + +Packages that only exist (or are only needed) on older TFMs. The polyfill major doesn't have to match any runtime band (older TFMs have no in-box equivalent), so pick the latest stable available: + +```xml + + + +``` + +Condition the *presence* of the reference, not the version. + +## How to categorize a package + +The named lists above aren't exhaustive. To classify a package you don't recognize, work through these steps in order: + +### 1. Read the nuget.org description + +Pure polyfills almost always say so explicitly. For example, `Microsoft.Bcl.TimeProvider`'s page reads: *"For apps targeting .NET 8 and newer versions, referencing this package is unnecessary, as the types it contains are already included in the .NET 8 and higher platform versions."* + +If the description says "for apps targeting .NET X and earlier" or "unnecessary on .NET X+", treat as polyfill candidate and continue to step 2 to confirm. If it makes no such claim and the package owner is Microsoft + a major number tracks the .NET release train, treat as runtime-aligned candidate. + +### 2. Inspect the package's `lib/` layout + +Look at the "Frameworks" tab on nuget.org or open the `.nupkg`: + +| `lib/` layout | Category | +|---|---| +| Only older TFM folders (`netstandard2.0`, `net462`) — no modern TFMs | Pure polyfill | +| `lib/net8.0/_._`, `lib/net9.0/_._` placeholders + real DLL only on older TFMs | Pure polyfill (no-op on modern TFMs) | +| Real DLLs in `lib/net8.0/`, `lib/net9.0/`, `lib/net10.0/`, differing per band | Runtime-aligned | +| Real DLLs on every TFM including older ones, single major doesn't track .NET releases | Independent | + +### 3. Check the release cadence + +- Runtime-aligned: new major every November in lockstep with .NET (8.0, 9.0, 10.0, ...) plus monthly servicing patches. +- Independent: releases on its own schedule, major doesn't correlate with .NET versions. +- Pure polyfill: usually freezes at one major and rarely bumps; new majors only to ride the build train. + +### 4. Functional test — remove the reference and rebuild + +The decisive test for the polyfill-vs-runtime-aligned boundary: remove the `PackageReference` on a modern TFM (e.g. net8) and build. + +- Builds clean → package was acting as a polyfill on that TFM. Confirm category 3. +- Fails with `CS0246`/`CS1061` (missing type or method) → the package contributes API the in-box BCL doesn't have. Treat as runtime-aligned (category 1), even if the description sounds polyfill-ish. + +### Beware hybrids + +Some packages look like polyfills but add API beyond the in-box BCL even on modern TFMs. Treat these like runtime-aligned packages. + +### When in doubt + +Treat as **runtime-aligned** (category 1) and reference on every TFM with per-TFM majors. + +- Cost of mis-classifying a true polyfill this way: a redundant `_._` asset at restore. Harmless. +- Cost of mis-classifying a hybrid as a pure polyfill: a compile break on the TFMs where you dropped the reference. + +## Why + +### Why latest minor/patch always + +`PackageReference` versions are minimums (`[X, ∞)`), so writing a stale minor or patch buys nothing for downgrade-safety and only loses fixes: + +- **Security**: BCL and Extensions packages ship CVE patches in minor/patch bumps. Pinning to an older patch means a customer who doesn't transitively pull a newer version stays on the vulnerable floor. +- **Bug fixes**: Same logic for non-security fixes. We have no reason to anchor customers to an older `8.0.0` when `8.0.5` is available. +- **NU1605 risk is small and easy to fix**: NU1605 fires on *any* downgrade, including minor/patch within the same major (e.g. our `8.0.5` transitive vs. a customer's direct `8.0.0`). In practice this is rare and trivial to resolve — the customer bumps their direct reference to a current patch. The cost of *not* tracking latest (stale security/bug fixes for every consumer who doesn't override) is larger than the cost of an occasional one-line bump in a consumer project. +- **Reduces noise from automated bumps**: Dependabot/Renovate PRs disappear if we already track latest. + +This rule applies to all three categories. The category decides the major; "latest" decides the minor and patch. + +### NuGet `PackageReference` versions are minimums, not pins + +`Version="X"` means `[X, ∞)`. The resolver picks the highest version requested across the graph, with one critical exception: a **direct** reference wins over a transitive one (nearest-wins). + +### NU1605 fires when the customer's direct version is *lower* than your transitive version + +If your library transitively requires `Microsoft.Extensions.Logging 10.0.0` and the consuming app has a direct ``, NuGet detects a downgrade and emits **NU1605**. In modern SDKs this is an **error**, not a warning — the customer's build fails. + +The reverse (customer's direct version higher than your transitive) resolves cleanly with no warning. + +### The asymmetry drives the rule + +| Your transitive version | Customer's direct version | Result | +|---|---|---| +| 10.x | 8.x | **NU1605 error** | +| 10.x | 10.x or higher | Clean | +| 8.x | 8.x or higher | Clean | + +Pinning runtime-aligned packages to the **band matching each TFM** means a net8 consumer transitively gets 8.x (no friction with their own 8.x reference), and a net10 consumer transitively gets 10.x. + +Pinning everything to the latest major (e.g. `10.x` unconditionally) forces every net8 customer to roll their direct references forward or hit NU1605. + +### Independent packages don't have this problem + +`Newtonsoft.Json 13.x`, `Azure.Identity 1.17.x`, etc. aren't tied to a runtime version. Customers don't have a "matching" version in mind, and the package's own multi-targeted assets handle TFM selection internally. One version is simpler and avoids needless conditional blocks. + +### Framework-provided assemblies win at runtime anyway + +On .NET 8+, packages like `System.Text.Json` and `System.Memory` are part of the shared framework. Even if you reference `System.Text.Json 9.0.0`, a net8 app uses the in-box net8 copy at runtime. Referencing 10.x on net8 just creates restore-graph noise with no runtime benefit — another reason to match the band. + +## Tradeoffs and alternatives considered + +### Per-TFM (the rule) vs. single lowest-LTS version + +A "pin everything to the lowest supported LTS major (e.g. 8.x everywhere) and bump only when that LTS drops" policy is also downgrade-safe and simpler in `Directory.Packages.props`. We rejected it because: + +- Customers on newer runtimes lose access to perf/feature work that landed in later package majors for packages that *aren't* fully overridden by the in-box shared framework (e.g. `Microsoft.Extensions.Caching.Memory`, `System.Configuration.ConfigurationManager`). +- The simplification is small: one extra conditional `ItemGroup` per runtime-aligned package. +- A coordinated bump when the floor LTS drops is a larger, riskier change than incremental per-TFM updates. + +Per-TFM keeps each runtime on its matching band and confines change to one `Update` line at a time. + +### Why no upper bounds + +Customers can transitively pull in a *higher* major than your reference. NuGet resolves nearest-wins with no warning, so a major that broke API can produce `MissingMethodException`/`TypeLoadException` at runtime rather than a restore-time failure. + +An upper bound (e.g. `Version="[8.0.0, 10.0.0)"`) would convert that runtime failure into a restore-time `NU1107` and is the only way to guarantee compatibility. We still avoid it because: + +- It blocks customers from rolling forward to fix CVEs or take perf wins in the newer major. +- It propagates conflicts deep into customer dependency graphs that we can't see. +- Microsoft's [library guidance](https://learn.microsoft.com/dotnet/standard/library-guidance/dependencies#nuget-dependency-version-ranges) explicitly says **AVOID upper bounds**, and the foundational Microsoft libraries (EF Core, ASP.NET Core, Aspire, Orleans, the BCL itself, Azure SDK) follow it. +- Exact pins (`[X]`) in Microsoft repos are reserved for host-coupled scenarios: Roslyn analyzers (`Microsoft.CodeAnalysis.*`), MSBuild API consumers (`Microsoft.Build`), VS extensibility. None apply to runtime libraries like SqlClient. + +We accept the SemVer-break risk in exchange for not breaking customer rollforward. If a specific package is known to break compatibility at a future major, document it in code review and revisit — don't blanket-bound. + +### Downgrade direction matters + +| Your transitive | Customer's direct | Result | +|---|---|---| +| Higher | Lower | **NU1605 error** (we cause this) | +| Lower | Higher | Clean restore, customer's wins | +| Equal | Equal | Clean | + +The asymmetry is the entire reason per-TFM matching works: it makes us the *lower* or *equal* for any customer who has aligned their own references with their runtime. + +## Checklist before changing a package version + +1. Is the package in the runtime-aligned list above? → use per-TFM conditional `PackageVersion Update`, latest minor/patch in each band. +2. Is it independent? → single `PackageVersion`, latest stable. +3. Is it a polyfill? → conditional `PackageReference` only on TFMs that need it, latest stable. +4. Always pick the latest available minor/patch within the chosen major. Don't carry forward a stale minor when bumping or adding. +5. Prefer Central Package Management (`Directory.Packages.props`) over per-project versions. +6. Never use exact-version (`[X]`) or upper-bound (`[X, Y)`) ranges on `PackageReference` unless you have a documented compatibility reason. + +## Sources + +- [NU1605 — Detected package downgrade](https://learn.microsoft.com/nuget/reference/errors-and-warnings/nu1605) +- [NuGet Package versioning — version ranges](https://learn.microsoft.com/nuget/concepts/package-versioning#version-ranges) +- [NuGet dependency resolution](https://learn.microsoft.com/nuget/concepts/dependency-resolution) +- [Library guidance — Dependencies](https://learn.microsoft.com/dotnet/standard/library-guidance/dependencies) +- [Library guidance — Cross-platform targeting](https://learn.microsoft.com/dotnet/standard/library-guidance/cross-platform-targeting) diff --git a/.github/instructions/ado-pipelines.instructions.md b/.github/instructions/ado-pipelines.instructions.md index 82b8046dee..688b57981a 100644 --- a/.github/instructions/ado-pipelines.instructions.md +++ b/.github/instructions/ado-pipelines.instructions.md @@ -1,195 +1,140 @@ --- applyTo: "eng/pipelines/**/*.yml" --- -# Azure DevOps Pipelines Guide - -## Overview - -This repository uses Azure DevOps Pipelines for CI/CD. The pipeline configurations are located in `eng/pipelines/`. - -**ADO Organization**: sqlclientdrivers -**ADO Project**: ADO.NET - -## Pipeline Structure - -``` -eng/pipelines/ -├── abstractions/ # Abstractions package pipelines -├── azure/ # Azure package pipelines -├── common/ # Shared templates -│ └── templates/ -│ ├── jobs/ # Reusable job templates -│ ├── stages/ # Reusable stage templates -│ └── steps/ # Reusable step templates -├── jobs/ # Top-level job definitions -├── libraries/ # Shared variable definitions -├── stages/ # Stage definitions -├── steps/ # Step definitions -├── variables/ # Variable templates -├── akv-official-pipeline.yml # AKV provider official/signing build -├── dotnet-sqlclient-ci-core.yml # Core CI pipeline (reusable) -├── dotnet-sqlclient-ci-package-reference-pipeline.yml # CI with package references -├── dotnet-sqlclient-ci-project-reference-pipeline.yml # CI with project references -├── dotnet-sqlclient-signing-pipeline.yml # Package signing pipeline -├── sqlclient-pr-package-ref-pipeline.yml # PR validation (package ref) -├── sqlclient-pr-project-ref-pipeline.yml # PR validation (project ref) -└── stress-tests-pipeline.yml # Stress testing -``` - -## Main Pipelines - -### CI Core Pipeline (`dotnet-sqlclient-ci-core.yml`) -Reusable core CI pipeline consumed by both project-reference and package-reference CI pipelines. Configurable parameters: - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `targetFrameworks` | Windows test frameworks | `[net462, net8.0, net9.0, net10.0]` | -| `targetFrameworksUnix` | Unix test frameworks | `[net8.0, net9.0, net10.0]` | -| `referenceType` | Project or Package reference | Required | -| `buildConfiguration` | Debug or Release | Required | -| `useManagedSNI` | Test with managed SNI | `[false, true]` | -| `testJobTimeout` | Test job timeout (minutes) | Required | -| `runAlwaysEncryptedTests` | Include AE tests | `true` | -| `enableStressTests` | Include stress test stage | `false` | - -### CI Reference Pipelines -- `dotnet-sqlclient-ci-project-reference-pipeline.yml` — Full CI using project references (builds from source) -- `dotnet-sqlclient-ci-package-reference-pipeline.yml` — Full CI using package references (tests against published NuGet packages) - -### PR Validation Pipelines -- `sqlclient-pr-project-ref-pipeline.yml` — PR validation with project references -- `sqlclient-pr-package-ref-pipeline.yml` — PR validation with package references - -These pipelines trigger on pull requests and run a subset of the full CI matrix to provide fast feedback. - -### Official/Signing Pipeline (`dotnet-sqlclient-signing-pipeline.yml`) -Signs and publishes NuGet packages. Used for official releases. Requires secure service connections and key vault access for code signing. - -### AKV Official Pipeline (`akv-official-pipeline.yml`) -Builds and signs the `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` add-on package separately from the main driver. Uses 1ES pipeline templates for compliance. - -### Stress Tests Pipeline (`stress-tests-pipeline.yml`) -Optional pipeline for long-running stress and endurance testing. Enabled via `enableStressTests` parameter in CI core. - -## Build Stages - -1. **build_abstractions_package_stage**: Build and pack abstractions -2. **build_sqlclient_package_stage**: Build main driver and AKV packages -3. **build_azure_package_stage**: Build Azure extensions package -4. **stress_tests_stage**: Optional stress testing -5. **run_tests_stage**: Execute all test suites +# Azure DevOps CI/CD Pipeline Guidelines + +## Purpose + +Rules and conventions for editing the Azure DevOps CI/CD pipelines that build, test, and validate Microsoft.Data.SqlClient. These pipelines live under `eng/pipelines/` (excluding `onebranch/`, which is covered by separate instructions). + +**ADO Organization**: sqlclientdrivers | **ADO Project**: ADO.NET + +## Pipeline Layout + +Two categories of pipelines exist in this repository: + +- **CI/PR pipelines** (`eng/pipelines/`) — Build, test, and validate on every push/PR +- **OneBranch pipelines** (`eng/pipelines/onebranch/`) — Official signing/release builds (separate instructions file) + +Top-level CI/PR pipeline files: +- `dotnet-sqlclient-ci-core.yml` — Reusable core template; all CI and PR pipelines extend this +- `dotnet-sqlclient-ci-package-reference-pipeline.yml` — CI with Package references (Release) +- `dotnet-sqlclient-ci-project-reference-pipeline.yml` — CI with Project references (Release) +- `sqlclient-pr-package-ref-pipeline.yml` — PR validation with Package references +- `sqlclient-pr-project-ref-pipeline.yml` — PR validation with Project references +- `ci/stress/sqlclient-ci-stress-pipeline.yml` — Stress test pipeline and templates + +Reusable templates are organized under: +- `common/templates/jobs/` — Job templates (`ci-build-nugets-job`, `ci-code-coverage-job`, `ci-run-tests-job`) +- `common/templates/stages/` — Stage templates (`ci-run-tests-stage`) +- `common/templates/steps/` — Step templates (build, test, config, publish) +- `jobs/` — Package-specific CI jobs (pack/test Abstractions, Azure, Logging, stress) +- `stages/` — Package-specific CI stages (generate secrets, build SqlServer/Logging/Abstractions/SqlClient/Azure, verify packages) +- `libraries/` — Shared variables (`ci-build-variables.yml`) +- `steps/` — SDK install steps + +## CI Core Template + +`dotnet-sqlclient-ci-core.yml` is the central orchestrator. All CI and PR pipelines extend it with different parameters. + +Key parameters: +- `referenceType` (required) — `Package` or `Project`; controls how sibling packages are referenced +- `buildConfiguration` (required) — `Debug` or `Release` +- `testJobTimeout` (required) — test job timeout in minutes +- `targetFrameworks` — Windows test TFMs; default `[net462, net8.0, net9.0, net10.0]` +- `targetFrameworksUnix` — Unix test TFMs; default `[net8.0, net9.0, net10.0]` +- `netcoreVersionTestUtils` — default runtime for shared test utilities; default `net10.0` +- `testSets` — test partitions; default `[1, 2, 3]` +- `useManagedSNI` — SNI variants to test; default `[false, true]` +- `runAlwaysEncryptedTests` — include AE test set; default `true` +- `runLegacySqlTests` — include SQL Server 2016/2017 manual-test legs; default `true` +- `debug` — enable debug output; default `false` +- `dotnetVerbosity` — build verbosity; default `normal` + +## Build Stage Order + +Stages execute in dependency order (Package reference mode requires artifacts from prior stages): +1. `generate_secrets` — Generate test secrets +2. `build_sqlserver_package_stage` — Build `Microsoft.SqlServer.Server` +3. `build_logging_package_stage` — Build Logging package +4. `build_abstractions_package_stage` — Build Abstractions (package mode depends on Logging) +5. `build_sqlclient_package_stage` — Build SqlClient and produce the AKV provider package +6. `build_azure_package_stage` — Build Azure extensions +7. `verify_nuget_packages_stage` — Verify NuGet package metadata +8. `ci_run_tests_stage` — Run MDS and AKV test suites + +Stress testing is no longer a stage threaded through `dotnet-sqlclient-ci-core.yml`; it lives under `eng/pipelines/ci/stress/` as a separate pipeline flow. + +When adding a new build stage, respect the dependency graph and pass artifact names/versions to downstream stages. + +## PR vs CI Pipeline Differences + +PR pipelines: +- Trigger on PRs targeting `release/7.1`; path filters vary by pipeline +- Exception: the legacy `sqlclient-pr-package-ref-pipeline.yml` has an empty branch include list, so it has no PR trigger and is manual-queue only +- Use reduced TFM matrix: `[net462, net8.0, net9.0]` (excludes net10.0) +- Timeout: 90 minutes +- Package-ref PR disables Always Encrypted tests in Debug config and also disables legacy SQL Server test legs to keep validation fast + +CI pipelines: +- Trigger on push to `release/7.1` (GitHub) and `internal/release/7.1` (ADO) with `batch: true` +- Exception: the legacy `dotnet-sqlclient-ci-package-reference-pipeline.yml` has its GitHub `release/7.1` push entry commented out, so only the ADO push trigger is active; its GitHub daily schedule still runs +- Scheduled daily builds are staggered to avoid main and other release-branch schedules (see individual pipeline files for cron times) +- Full TFM matrix including net10.0 test legs and legacy SQL Server manual-test coverage ## Test Configuration -### Test Sets -Tests are divided into sets for parallelization: -- `TestSet=1` — First partition of tests -- `TestSet=2` — Second partition -- `TestSet=3` — Third partition -- `TestSet=AE` — Always Encrypted tests - -### Test Filters -Tests use category-based filtering. The default filter excludes both `failing` and `flaky` tests: -``` -category!=failing&category!=flaky -``` - -Category values: -- `nonnetfxtests` — Excluded on .NET Framework -- `nonnetcoreapptests` — Excluded on .NET Core -- `nonwindowstests` — Excluded on Windows -- `nonlinuxtests` — Excluded on Linux -- `failing` — Known permanent failures (excluded from all runs) -- `flaky` — Intermittently failing tests (quarantined, run separately) - -### Flaky Test Quarantine in Pipelines -Quarantined tests (`[Trait("Category", "flaky")]`) run in **separate pipeline steps** after the main test steps. This ensures: -- Main test runs are **not blocked** by intermittent failures -- Flaky tests are still **monitored** for regression or resolution -- Code coverage is **not collected** for flaky test runs -- Results appear in pipeline output for visibility - -The quarantine steps are configured in: -- `eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml` -- `eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml` -- `eng/pipelines/common/templates/steps/run-all-tests-step.yml` - -### Test Timeout -All test runs use `--blame-hang-timeout 10m` (configured in `build.proj`). Tests exceeding 10 minutes are killed and reported as failures. - -### SNI Testing -The `useManagedSNI` parameter controls testing with: -- Native SNI (`false`) - Windows native library -- Managed SNI (`true`) - Cross-platform managed implementation +Test partitioning: +- Tests split into `TestSet=1`, `TestSet=2`, `TestSet=3` for parallelization +- `TestSet=AE` — Always Encrypted tests (controlled by `runAlwaysEncryptedTests`) -## Variables - -### Build Variables (`ci-build-variables.yml`) -Common build configuration: -- Package versions -- Build paths -- Signing configuration - -### Runtime Variables -Set via pipeline parameters or UI: -- `Configuration` - Debug/Release -- `Platform` - AnyCPU/x86/x64 -- `TF` - Target framework - -## Creating Pipeline Changes +Test filters — default excludes `failing` and `flaky` categories: +- `failing` — known permanent failures, always excluded +- `flaky` — intermittent failures, quarantined in separate pipeline steps +- `nonnetfxtests` / `nonnetcoreapptests` — platform-specific exclusions +- `nonwindowstests` / `nonlinuxtests` — OS-specific exclusions -### Adding New Test Categories -1. Add category attribute to tests: `[Category("newcategory")]` -2. Update filter expressions in test job templates -3. Document category purpose in test documentation +Flaky test quarantine: +- Quarantined tests (`[Trait("category", "flaky")]`) run in separate steps after main tests +- Main test runs are not blocked by flaky failures +- No code coverage collected for flaky runs +- Configured in `common/templates/steps/build-and-run-tests-netcore-step.yml`, `build-and-run-tests-netfx-step.yml`, and `run-all-tests-step.yml` -### Adding New Pipeline Parameters -1. Define parameter in appropriate `.yml` file -2. Add to parameter passing in calling templates -3. Document in this file +SNI testing — `useManagedSNI` controls testing with native SNI (`false`) or managed SNI (`true`) -### Modifying Build Steps -1. Changes should be made in template files for reusability -2. Test changes locally when possible -3. Submit as PR - validation will run +Test timeout — `--blame-hang-timeout 10m` (configured in `build.proj` and threaded through CI test steps); tests exceeding 10 minutes are killed -## Best Practices - -### Template Design -- Use templates for reusable definitions -- Pass parameters explicitly (avoid global variables) -- Use descriptive stage/job/step names - -### Variable Management -- Use template variables for shared values -- Use pipeline parameters for per-run configuration -- Avoid hardcoding versions (use Directory.Packages.props) +## Variables -### Test Infrastructure -- Ensure tests are properly categorized -- Handle test configuration files properly -- Use test matrix for cross-platform coverage +- All CI build variables centralized in `libraries/ci-build-variables.yml` +- Package versions use `-ci` suffix (e.g., `7.0.0.$(Build.BuildNumber)-ci`) +- `assemblyBuildNumber` derived from first segment of `Build.BuildNumber` (16-bit safe) +- `localFeedPath` = `$(Build.SourcesDirectory)/packages` — local NuGet feed for inter-package deps +- `packagePath` = `$(Build.SourcesDirectory)/output` — NuGet pack output -## Troubleshooting +## Variable Naming — Avoid `{COMMAND}ARGUMENTS` Names -### Common Issues -1. **Test failures due to missing config**: Ensure `config.json` exists -2. **Platform-specific failures**: Check platform exclusion categories -3. **Timeout issues**: Increase `testJobTimeout` parameter +The dotnet CLI (via System.CommandLine) reads environment variables named `{COMMAND}ARGUMENTS` and silently injects their content into the parsed arguments for that subcommand. Because Azure DevOps automatically exposes all pipeline variables as uppercased environment variables, a pipeline variable named `runArguments` becomes `RUNARGUMENTS`, which `dotnet run` reads and injects into the application's `args[]` — bypassing the `--` separator. -### Debugging Pipelines -- Enable debug mode via `debug: true` parameter -- Use `dotnetVerbosity: diagnostic` for detailed output -- Check build logs in Azure DevOps +**Forbidden variable names** (any casing): +- `runArguments` — injected into `dotnet run` +- `buildArguments` — injected into `dotnet build` +- `testArguments` — injected into `dotnet test` +- Any name matching `{dotnet-subcommand}Arguments` -## Security Considerations +**Use instead**: `dotnetBuildOpts`, `dotnetRunOpts`, `stressTestArgs`, or other names that do not match the `{COMMAND}ARGUMENTS` pattern. -- Pipelines use service connections for artifact publishing -- Signing uses secure key vault integration -- Sensitive configuration should use pipeline secrets -- Never commit credentials in pipeline files +This affects ALL .NET SDK versions (8.0+). The injection is invisible in `[command]` log lines, making it extremely hard to diagnose. The only symptom is the application receiving unexpected arguments. -## Related Documentation +## Conventions When Editing Pipelines -- [BUILDGUIDE.md](../../BUILDGUIDE.md) - Local build instructions -- [Azure DevOps Documentation](https://learn.microsoft.com/azure/devops/pipelines/) +- Always use templates for reusable logic — do not inline complex steps +- Pass parameters explicitly; avoid relying on global variables +- Use descriptive stage/job/step display names +- When adding parameters, define them in the core template and thread through calling pipelines +- When adding test categories, update filter expressions in test step templates +- PR pipelines should run a minimal matrix for fast feedback +- Test changes via PR pipeline first — validation runs automatically +- Enable `debug: true` and `dotnetVerbosity: diagnostic` for troubleshooting +- Never commit credentials or secrets in pipeline files +- Signing and release are handled by OneBranch pipelines — not these CI/PR pipelines diff --git a/.github/instructions/ado-work-items-markdown.instructions.md b/.github/instructions/ado-work-items-markdown.instructions.md new file mode 100644 index 0000000000..2a70fd124a --- /dev/null +++ b/.github/instructions/ado-work-items-markdown.instructions.md @@ -0,0 +1,139 @@ +--- +applyTo: "**" +--- +# Azure DevOps Work Items: Markdown Description Rules + +Use this guide whenever creating or updating Azure DevOps work items that include rich text in `System.Description`. + +## Goals + +- Ensure descriptions render as Markdown (not HTML/plain text) +- Preserve newline characters and list structure +- Verify work items after every batch update + +## Required Behavior + +1. Always use `az rest` for description content-type changes. +2. Use `application/json-patch+json` for PATCH requests. +3. Set `multilineFieldsFormat.System.Description` to `markdown`. +4. Preserve exact newlines in the Markdown body. +5. Verify both format and newline integrity after updates. + +## Authentication and Resource + +Use Azure DevOps resource audience when calling `az rest`: + +- Resource: `499b84ac-1321-427f-aa17-267ca6975798` + +Example auth check: + +```bash +az rest \ + --method GET \ + --resource 499b84ac-1321-427f-aa17-267ca6975798 \ + --url "https://dev.azure.com//_apis/projects?api-version=7.1-preview.4" +``` + +Note: API versions can differ by endpoint. The examples below use `7.1-preview.3` for work item PATCH calls, while this auth check uses `7.1-preview.4`. + +## Safe Update Pattern (Prevents Type/Value Errors) + +Some work items reject a direct type switch unless a valid value is provided. Use this two-step process: + +### Step 1: Capture current description + +```bash +az boards work-item show --id | jq -j '.fields["System.Description"] // ""' > ./desc-backup.txt +``` + +> **NOTE**: Writing to a file preserves all characters including trailing newlines. +> Command substitution (`$(...)`) silently strips trailing newlines, which would +> corrupt multi-line Markdown content. + +### Step 2: Force markdown type with temporary empty value + +```bash +PATCH_STEP1_FILE="${PATCH_STEP1_FILE:-./patch-step1.json}" + +jq -n '[ + {"op":"replace","path":"/fields/System.Description","value":""}, + {"op":"replace","path":"/multilineFieldsFormat/System.Description","value":"markdown"} +]' >"$PATCH_STEP1_FILE" + +az rest \ + --method PATCH \ + --resource 499b84ac-1321-427f-aa17-267ca6975798 \ + --url "https://dev.azure.com///_apis/wit/workitems/?api-version=7.1-preview.3" \ + --headers "Content-Type=application/json-patch+json" \ + --body @"$PATCH_STEP1_FILE" +``` + +### Step 3: Restore exact Markdown text + +```bash +PATCH_STEP2_FILE="${PATCH_STEP2_FILE:-./patch-step2.json}" + +jq -n --rawfile d ./desc-backup.txt '[ + {"op":"replace","path":"/fields/System.Description","value":$d} +]' >"$PATCH_STEP2_FILE" + +az rest \ + --method PATCH \ + --resource 499b84ac-1321-427f-aa17-267ca6975798 \ + --url "https://dev.azure.com///_apis/wit/workitems/?api-version=7.1-preview.3" \ + --headers "Content-Type=application/json-patch+json" \ + --body @"$PATCH_STEP2_FILE" +``` + +## Newline Integrity Checks + +After updates, confirm newline characters are still present and structure was not flattened. + +### Check format and description sample + +```bash +az boards work-item show --id | jq '.multilineFieldsFormat, .fields["System.Description"][0:200]' +``` + +Expected: + +- `multilineFieldsFormat.System.Description == "markdown"` +- Description text contains `\n` where line breaks are expected + +### Check line count did not collapse + +```bash +az boards work-item show --id \ +| jq -r '.fields["System.Description"]' \ +| awk 'END { print NR }' +``` + +If a multi-line description unexpectedly returns `1`, newline content was likely lost. + +## Batch Verification Script + +Use this after bulk updates: + +```bash +python3 - <<'PY' +import json, subprocess +ids = [12345, 12346] # replace with your target IDs +bad = [] +for i in ids: + out = subprocess.check_output(["az", "boards", "work-item", "show", "--id", str(i)], text=True) + j = json.loads(out) + fmt = (j.get("multilineFieldsFormat") or {}).get("System.Description") + desc = j.get("fields", {}).get("System.Description") or "" + if fmt != "markdown" or "\n" not in desc: + bad.append((i, fmt, "has_newlines" if "\n" in desc else "missing_newlines")) +print("noncompliant:", len(bad)) +for row in bad: + print(row) +PY +``` + +## Common Failure Modes + +- `401` or `TF400813`: wrong token audience or insufficient auth context +- `Content-Type ... not supported`: must use `application/json-patch+json` +- `type changed without a value`: use two-step pattern (empty + markdown type, then restore text) diff --git a/.github/instructions/agentic-workflows.instructions.md b/.github/instructions/agentic-workflows.instructions.md new file mode 100644 index 0000000000..6a42c0dce3 --- /dev/null +++ b/.github/instructions/agentic-workflows.instructions.md @@ -0,0 +1,49 @@ +--- +applyTo: ".github/workflows/**/*.md" +description: Rules for editing gh-aw agentic workflow Markdown files. +--- + +# Agentic Workflow Edit Rules (`gh aw`) + +This repository authors GitHub Actions agentic workflows in Markdown using +[`gh aw`](https://github.com/githubnext/gh-aw). Each workflow `.md` file under +`.github/workflows/` compiles to a sibling `.lock.yml`, and **only the +`.lock.yml` is executed by GitHub Actions at runtime.** + +## Mandatory rule + +Whenever you create, edit, rename, or delete a file matching +`.github/workflows/**/*.md`, you **MUST**, in the **same commit / PR**: + +1. Run `gh aw compile` from the repository root. +2. Stage and commit the regenerated sibling `.lock.yml`. +3. If you deleted a workflow `.md`, also delete its `.lock.yml`. + +If the `.lock.yml` is stale or missing, the workflow fails at runtime +(see PR #4279 for the exact failure mode). The +`Verify gh aw lock files` CI check will block the PR in that case. + +## How to verify locally + +```bash +gh aw compile +git status # both the .md and .lock.yml should appear +gh aw compile # second run must be a no-op (clean diff) +``` + +## Code-review checklist + +When reviewing a PR that touches `.github/workflows/**/*.md`: + +- [ ] A matching `.lock.yml` is updated in the same PR. +- [ ] `gh aw compile` produces no further diff on top of the PR. +- [ ] If new tools, network endpoints, or permissions are added in the `.md`, + they are present in the regenerated `.lock.yml`. + +## Out of scope + +- Do **not** hand-edit `.lock.yml` files. They are generated; edit the `.md` + source and recompile. +- For deeper authoring guidance (creating, debugging, upgrading workflows), + invoke the `agentic-workflows` agent at + `.github/agents/agentic-workflows.agent.md`. diff --git a/.github/instructions/architecture.instructions.md b/.github/instructions/architecture.instructions.md index 3314babb7c..2fd30c9f55 100644 --- a/.github/instructions/architecture.instructions.md +++ b/.github/instructions/architecture.instructions.md @@ -34,10 +34,11 @@ src/ ## Unified Project Model ### Architecture Goal -The driver is transitioning away from separate `netfx/` and `netcore/` project files toward a **single unified project** at `src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj`. This project multi-targets all supported frameworks from one codebase: +The driver is transitioning away from separate `netfx/` and `netcore/` project files toward a **single unified project** at `src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj`. This project targets the modern .NET TFMs on every host and conditionally adds .NET Framework on Windows: ```xml -net462;net8.0;net9.0 +net8.0;net9.0 +$(TargetFrameworks);net462 ``` **All new code MUST go into `src/Microsoft.Data.SqlClient/src/`**. Do NOT add files to the legacy `netcore/src/` or `netfx/src/` directories. @@ -48,7 +49,7 @@ The `netcore/` and `netfx/` directories are legacy artifacts from the old dual-p - `netcore/ref/` and `netfx/ref/` — **STILL ACTIVE**. Reference assemblies remain in these directories and define the public API surface for each target framework. ### OS Targeting with `TargetOs` -The unified project uses a `TargetOs` MSBuild property to handle OS-specific compilation: +The unified project uses a `TargetOs` build property to handle OS-specific compilation: ```xml @@ -82,7 +83,7 @@ When writing code that differs by platform, use these preprocessor directives: | `#if _UNIX` | Code for Unix/Linux/macOS OS (any framework) | Guidelines: -1. All code must compile for all target frameworks (`net462`, `net8.0`, `net9.0`) +1. All code must compile for the TFMs supported by the current target OS: `net8.0`/`net9.0` everywhere, plus `net462` on Windows builds 2. Use `#if NETFRAMEWORK` or `#if NET` for framework-specific code paths 3. Use `#if _WINDOWS` or `#if _UNIX` for OS-specific code paths 4. Avoid APIs that don't exist on a target platform without conditional compilation @@ -104,11 +105,15 @@ The `ref/` directories define the public API surface: **IMPORTANT**: Any public API changes MUST update the corresponding reference assembly in the appropriate `ref/` directory. ### Build Output -Build artifacts are organized by framework and OS: +Build artifacts are organized by reference mode, configuration, OS, and framework: ``` -artifacts/Microsoft.Data.SqlClient/{Configuration}/{TargetOs}/{TargetFramework}/ +artifacts/Microsoft.Data.SqlClient/{ReferenceType}-{Configuration}/{NormalizedTargetOs}/{TargetFramework}/ ``` +`ReferenceType` is a first-class build dimension in this branch. Local and CI builds may run in: +- `Project` mode — sibling packages referenced as projects +- `Package` mode — sibling packages restored from locally produced NuGet packages + ## SNI (SQL Server Network Interface) Layer Two implementations exist: diff --git a/.github/instructions/connection-pooling.instructions.md b/.github/instructions/connection-pooling.instructions.md index 6464fdedd0..95019b996d 100644 --- a/.github/instructions/connection-pooling.instructions.md +++ b/.github/instructions/connection-pooling.instructions.md @@ -124,13 +124,6 @@ private readonly ChannelWriter _idleConnectionWriter; await _idleConnectionReader.WaitToReadAsync(token); ``` -### Sync Over Async Protection -```csharp -// Prevent thread pool starvation -private static SemaphoreSlim _syncOverAsyncSemaphore = - new(Math.Max(1, Environment.ProcessorCount / 2)); -``` - ## Best Practices ### Application Design diff --git a/.github/instructions/features.instructions.md b/.github/instructions/features.instructions.md index 3ecba2cc4e..34262b8db6 100644 --- a/.github/instructions/features.instructions.md +++ b/.github/instructions/features.instructions.md @@ -246,6 +246,7 @@ AppContext switches allow runtime behavior changes without modifying connection | `Switch.Microsoft.Data.SqlClient.EnableMultiSubnetFailoverByDefault` | `false` | Sets `MultiSubnetFailover=true` as the default for all connections | | `Switch.Microsoft.Data.SqlClient.EnableUserAgent` | varies | Controls sending user agent information to SQL Server | | `Switch.Microsoft.Data.SqlClient.IgnoreServerProvidedFailoverPartner` | `false` | Ignores failover partner information sent by the server | +| `Switch.Microsoft.Data.SqlClient.UseLegacyFailoverAlternationOnLoginSqlErrors` | `false` | Restores legacy `LoginWithFailover` alternation for login-phase SQL errors when parser state is not `Closed` | | `Switch.Microsoft.Data.SqlClient.LegacyRowVersionNullBehavior` | `false` | Restores legacy null handling for rowversion columns | | `Switch.Microsoft.Data.SqlClient.LegacyVarTimeZeroScaleBehaviour` | `false` | Restores legacy zero-scale behavior for time/datetime2/datetimeoffset | | `Switch.Microsoft.Data.SqlClient.MakeReadAsyncBlocking` | `false` | Makes ReadAsync behave synchronously (legacy compat) | diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 1c848e2493..02717a3a13 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -1,534 +1,179 @@ --- applyTo: "eng/pipelines/**/*.yml" --- -# Multi-Product Azure DevOps Pipeline in dotnet/sqlclient — Design Specification - -## 1. Overview - -This document describes the design of the unified Azure DevOps YAML pipeline that builds, signs, packages, and optionally releases six NuGet packages with interdependencies. The pipeline uses **stages** and **jobs** to maximize parallelism while respecting dependency order. It comprises five stages: three build stages, a validation stage, and an on-demand release stage. - -Two pipeline variants exist from the same stage/job structure: - -| Pipeline | Template | Trigger | Purpose | -|----------|----------|---------|---------| -| `dotnet-sqlclient-official-pipeline.yml` | `OneBranch.Official.CrossPlat.yml` | CI + scheduled | Production-signed builds | -| `dotnet-sqlclient-non-official-pipeline.yml` | `OneBranch.NonOfficial.CrossPlat.yml` | Manual only | Validation / test builds (release in dry-run mode) | - -Both pipelines use the **OneBranch (1ES) governed template** infrastructure and share identical stage definitions, job templates, and variable chains. - ---- - -## 2. Products and Dependencies - -| # | Package | Dependencies | -|---|---------|-------------| -| 1 | `Microsoft.SqlServer.Server` | — | -| 2 | `Microsoft.Data.SqlClient.Internal.Logging` | — | -| 3 | `Microsoft.Data.SqlClient.Extensions.Abstractions` | `Internal.Logging` | -| 4 | `Microsoft.Data.SqlClient` | `Internal.Logging`, `Extensions.Abstractions` | -| 5 | `Microsoft.Data.SqlClient.Extensions.Azure` | `Extensions.Abstractions`, `Internal.Logging` | -| 6 | `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | `SqlClient`, `Internal.Logging` | - ---- - -## 3. Pipeline Flow — Sequence Diagram - -```mermaid -sequenceDiagram - participant T as Trigger / User - participant P as Pipeline Orchestrator - participant B1a as Job: Build Internal.Logging - participant B1c as Job: Build SqlServer.Server - participant B1b as Job: Build Extensions.Abstractions - participant B2a as Job: Build SqlClient - participant B2b as Job: Build Extensions.Azure - participant V as Job: Validate MDS Package - participant B3 as Job: Build AKV Provider - participant R as Stage: Release - - Note over T,R: ══════ BUILD & SIGN PHASE ══════ - - T->>P: Pipeline triggered (CI / Scheduled / Manual) - - Note over P: Stage 1 — build_independent (parallel, no deps) - - par Stage 1 jobs (parallel) - P->>B1a: Build DLLs → ESRP sign DLLs → Pack → ESRP sign NuGet (Logging) - B1a-->>P: ✅ Signed .nupkg - and - P->>B1c: Build + ESRP sign + pack SqlServer.Server - B1c-->>P: ✅ Signed .nupkg - end - - Note over P: Stage 2 — build_abstractions (dependsOn: build_independent) - - P->>B1b: Build DLLs → ESRP sign DLLs → Pack → ESRP sign NuGet (Abstractions) - Note right of B1b: Downloads: Internal.Logging artifact - B1b-->>P: ✅ Signed .nupkg - - Note over P: Stage 3 — build_dependent (dependsOn: build_abstractions) - - par Stage 3 jobs (parallel) - P->>B2a: Build + ESRP sign + pack SqlClient - Note right of B2a: Downloads: Internal.Logging,
Extensions.Abstractions artifacts - B2a-->>P: ✅ Signed .nupkg + .snupkg - and - P->>B2b: Build DLLs → ESRP sign DLLs → Pack → ESRP sign NuGet (Azure) - Note right of B2b: Downloads: Extensions.Abstractions,
Internal.Logging artifacts - B2b-->>P: ✅ Signed .nupkg - end - - Note over P: Validation + Stage 4 (both dependsOn: build_dependent, run in parallel) - - par Validation and Stage 4 (parallel) - P->>V: Validate signed MDS package - V-->>P: ✅ Package validation passed - and - P->>B3: Build + ESRP sign + pack AKV Provider - Note right of B3: Downloads: SqlClient,
Internal.Logging artifacts - B3-->>P: ✅ Signed .nupkg - end - - Note over T,R: ══════ RELEASE PHASE (on-demand) ══════ - - alt At least one release parameter is true - P->>R: Stage: release (dependsOn: conditional on build stages) - Note right of R: ADO Environment Approval
(NuGet-Production environment) - R-->>P: ✅ Approved - Note right of R: Publish selected packages
via NuGetCommand@2 - R-->>P: ✅ Published to NuGet - else No release parameters set - Note over P: Release stage skipped - end - - Note over T,R: Pipeline complete 🎉 -``` - ---- - -## 4. Stage Design - -### 4.1 Build Phase - -The build phase runs automatically on every CI trigger, scheduled run, or manual queue. It is divided into four build stages plus a validation stage, based on the dependency graph. - -#### Stage 1 — `build_independent`: Independent Packages (no dependencies) - -| Job Template | Package | Build Target | Condition | -|--------------|---------|--------------|-----------| -| `build-signed-csproj-package-job.yml` | `Microsoft.Data.SqlClient.Internal.Logging` | `BuildLogging` / `PackLogging` | `buildAKVProvider OR buildSqlClient` | -| `build-signed-csproj-package-job.yml` | `Microsoft.SqlServer.Server` | `PackSqlServer` | `buildSqlServerServer` | - -- **`dependsOn`**: none -- **Parallelism**: Jobs run in parallel (depending on which are enabled) -- **Conditional builds**: Each job is wrapped with compile-time `${{ if }}` conditionals based on build parameters -- csproj-based jobs (`build-signed-csproj-package-job.yml`) perform: **Build DLLs → ESRP DLL signing → NuGet pack (NoBuild=true) → ESRP NuGet signing** → publish artifact - -#### Stage 2 — `build_abstractions`: Abstractions Package (depends on Stage 1) - -| Job Template | Package | Build Target | Artifact Dependencies | -|--------------|---------|--------------|----------------------| -| `build-signed-csproj-package-job.yml` | `Microsoft.Data.SqlClient.Extensions.Abstractions` | `BuildAbstractions` / `PackAbstractions` | `Internal.Logging` | - -- **Stage condition**: `buildSqlClient = true` (entire stage is excluded when false) -- **`dependsOn`**: `build_independent` -- Downloads `Microsoft.Data.SqlClient.Internal.Logging.nupkg` (from Stage 1) pipeline artifact - -#### Stage 3 — `build_dependent`: Core Packages (depend on Stage 2) - -| Job Template | Package | Build Target | Artifact Dependencies | -|--------------|---------|--------------|----------------------| -| `build-signed-package-job.yml` | `Microsoft.Data.SqlClient` | *(nuspec-based)* | `Internal.Logging`, `Extensions.Abstractions` | -| `build-signed-csproj-package-job.yml` | `Microsoft.Data.SqlClient.Extensions.Azure` | `BuildAzure` / `PackAzure` | `Extensions.Abstractions`, `Internal.Logging` | - -- **Stage condition**: `buildSqlClient = true` (entire stage is excluded when false) -- **`dependsOn`**: `build_abstractions` -- **Parallelism**: Both jobs run in parallel -- The MDS (SqlClient) job also publishes symbol packages (`.snupkg`) when `publishSymbols` is true -- All jobs configure APIScan with job-level `ob_sdl_apiscan_*` variables targeting package-specific folders - -#### Stage 4 — `build_addons`: Add-on Packages (depend on Stage 3) - -| Job Template | Package | Artifact Dependencies | -|--------------|---------|----------------------| -| `build-akv-official-job.yml` | `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | `SqlClient`, `Internal.Logging` | - -- **Stage condition**: `buildAKVProvider AND buildSqlClient` (both must be true) -- **`dependsOn`**: `build_dependent` -- Downloads `Microsoft.Data.SqlClient.nupkg` (from Stage 3) and `Microsoft.Data.SqlClient.Internal.Logging.nupkg` (from Stage 1) pipeline artifacts -- Uses separate ESRP signing credentials (`Signing`-prefixed variables from `esrp-variables-v2` group) - -### 4.2 Validation Stage — `mds_package_validation` - -Validates the signed MDS (SqlClient) package after Stage 3 completes. - -- **Stage condition**: `buildSqlClient = true` -- **`dependsOn`**: `build_dependent` -- Runs in parallel with Stage 4 (`build_addons`) -- Uses `validate-signed-package-job.yml` template -- Downloads the `drop_build_dependent_build_signed_package` artifact and validates against `CurrentNetFxVersion` (default: `net462`) - -### 4.3 Release Phase — `release` - -The release stage is gated and only executes on demand when at least one release parameter is set to `true` at queue time. - -- **`dependsOn`**: Conditional based on which build stages are enabled: - - `build_independent` (when releasing SqlServer.Server or Logging) - - `build_abstractions` (when releasing Abstractions) - - `build_dependent`, `mds_package_validation` (when `buildSqlClient = true`) - - `build_addons` (when `buildAKVProvider AND buildSqlClient`) -- **Gate**: ADO Environment approvals (official pipeline only): - - Official: `NuGet-Production` environment with configured approvals - - Non-Official: `NuGet-DryRun` environment (no approvals, validation only) -- **Package selection**: Controlled by 6 runtime boolean parameters (see Section 5.2) -- **Stage condition**: The entire stage is skipped unless at least one release parameter is `true`: - ```yaml - - ${{ if or(parameters.releaseSqlServerServer, parameters.releaseLogging, ...) }}: - - stage: release - ``` -- **Publish jobs**: Each package has a conditional publish job that is included at compile time only when its parameter is `true`: - ```yaml - - ${{ if eq(parameters.releaseXxx, true) }}: - - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self - ``` -- **Environment variables**: Stage sets `ob_release_usedeploymentjob: true` for OneBranch integration: - - Official: `ob_release_environment: 'NuGet-Production'` - - Non-Official: `ob_release_environment: 'NuGet-DryRun'` - -#### Artifact → Publish Job Mapping - -| Package | Artifact Name | Publish Job | -|---------|---------------|-------------| -| `Microsoft.SqlServer.Server` | `drop_build_independent_build_package_SqlServer` | `publish_SqlServer_Server` | -| `Microsoft.Data.SqlClient.Internal.Logging` | `drop_build_independent_build_package_Logging` | `publish_Logging` | -| `Microsoft.Data.SqlClient.Extensions.Abstractions` | `drop_build_abstractions_build_package_Abstractions` | `publish_Abstractions` | -| `Microsoft.Data.SqlClient` | `drop_build_dependent_build_package_SqlClient` | `publish_SqlClient` | -| `Microsoft.Data.SqlClient.Extensions.Azure` | `drop_build_dependent_build_package_Azure` | `publish_Extensions_Azure` | -| `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | `drop_build_addons_buildSignedAkvPackage` | `publish_AKVProvider` | - -Each publish job uses the reusable `publish-nuget-package-job.yml` template, which downloads the artifact and pushes `.nupkg`/`.snupkg` files via `NuGetCommand@2` with an external feed service connection. - -#### Dry-Run Mode - -Two ADO environments control release behavior: - -| Environment | Pipeline | Behavior | -|------------|----------|----------| -| `NuGet-DryRun` | Non-Official | Validation only — packages are never pushed | -| `NuGet-Production` | Official | Real releases with approval gate | - -**Non-official pipeline**: Always runs in dry-run mode. There is no `releaseDryRun` parameter — `dryRun: true` is hardcoded in every publish job. This prevents accidental publication from validation builds. - -**Official pipeline**: Exposes a `releaseDryRun` parameter (default: `true` for safety). When enabled, the template downloads artifacts and lists the `.nupkg`/`.snupkg` files that *would* be published but skips the actual `NuGetCommand@2` push. Set `releaseDryRun: false` to perform real pushes after final validation. - ---- - -## 5. Runtime Parameters - -### 5.1 Build Parameters - -The pipeline exposes the following parameters at queue time: - -```yaml -parameters: - - name: debug - displayName: 'Enable debug output' - type: boolean - default: false - - - name: publishSymbols - displayName: 'Publish symbols' - type: boolean - default: false - - - name: CurrentNetFxVersion - displayName: 'Lowest supported .NET Framework version (MDS validation)' - type: string - default: 'net462' - - - name: isPreview - displayName: 'Is this a preview build?' - type: boolean - default: false - - - name: testJobTimeout - displayName: 'Test job timeout (in minutes)' - type: number - default: 60 - - # Build parameters — control which packages to build - - name: buildSqlServerServer - displayName: 'Build Microsoft.SqlServer.Server' - type: boolean - default: true - - - name: buildSqlClient - displayName: 'Build Microsoft.Data.SqlClient and Extensions' - type: boolean - default: true - - - name: buildAKVProvider - displayName: 'Build Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' - type: boolean - default: true -``` - -The `isPreview` parameter controls version resolution — when `true`, each package uses its preview version (e.g., `loggingPackagePreviewVersion`) instead of the GA version (e.g., `loggingPackageVersion`). All versions are defined in the centralized `libraries/common-variables.yml`. - -The build parameters enable selective package building: -- `buildSqlServerServer` — controls SqlServer.Server build job -- `buildSqlClient` — controls MDS, Extensions.Azure, Abstractions, Logging (when AKV is disabled), and validation stages -- `buildAKVProvider` — controls AKV Provider build (also requires `buildSqlClient=true`) and Logging (when SqlClient is disabled) - -When set to `false`, the respective jobs/stages are excluded at compile-time using `${{ if }}` conditionals. This allows faster pipeline runs when only certain packages need to be built. - -### 5.2 Release Parameters - -Six boolean parameters control selective package release. All default to `false` so the release stage is skipped on normal CI/scheduled builds: - -```yaml -parameters: - - name: releaseSqlServerServer - displayName: 'Release Microsoft.SqlServer.Server' - type: boolean - default: false - - - name: releaseLogging - displayName: 'Release Microsoft.Data.SqlClient.Internal.Logging' - type: boolean - default: false - - - name: releaseAbstractions - displayName: 'Release Microsoft.Data.SqlClient.Extensions.Abstractions' - type: boolean - default: false - - - name: releaseSqlClient - displayName: 'Release Microsoft.Data.SqlClient' - type: boolean - default: false - - - name: releaseAzure - displayName: 'Release Microsoft.Data.SqlClient.Extensions.Azure' - type: boolean - default: false - - - name: releaseAKVProvider - displayName: 'Release Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' - type: boolean - default: false -``` - -#### Dry-Run Parameter (Official Pipeline Only) - -The **official pipeline** includes a `releaseDryRun` parameter that defaults to `true` for safety: - -```yaml - - name: releaseDryRun - displayName: 'Release Dry Run (do not push to NuGet)' - type: boolean - default: true # safety default — must explicitly disable for real releases -``` - -When `releaseDryRun: true`, publish jobs download artifacts and list packages but skip actual NuGet push. Set to `false` for production releases. - -> **Note**: The non-official pipeline does **not** expose this parameter — dry-run mode is hardcoded and cannot be disabled. - ---- - -## 6. Variable & Version Management - -### 6.1 Variable Chain - -Variables are defined in a layered template chain. All variable groups live inside the templates — none are declared inline at the pipeline level: - -``` -dotnet-sqlclient-official-pipeline.yml - └─ libraries/variables.yml - └─ libraries/build-variables.yml - ├─ group: 'Release Variables' - ├─ group: 'Symbols publishing' ← SymbolsPublishServer, SymbolsPublishTokenUri, etc. - └─ libraries/common-variables.yml - ├─ group: 'ESRP Federated Creds (AME)' ← ESRP signing credentials - ├─ SymbolServer / SymbolTokenUri aliases ← mapped from Symbols publishing group - └─ all package versions, paths, build variables -``` - -### 6.2 Package Version Variables - -All package versions are centralized in `libraries/common-variables.yml`: - -| Package | GA Version Var | Preview Version Var | Assembly Version Var | -|---------|---------------|--------------------|--------------------| -| Logging | `loggingPackageVersion` | `loggingPackagePreviewVersion` | `loggingAssemblyFileVersion` | -| Abstractions | `abstractionsPackageVersion` | `abstractionsPackagePreviewVersion` | `abstractionsAssemblyFileVersion` | -| SqlServer.Server | `sqlServerPackageVersion` | `sqlServerPackagePreviewVersion` | `sqlServerAssemblyFileVersion` | -| SqlClient (MDS) | `mdsPackageVersion` | `mdsPackagePreviewVersion` | `mdsAssemblyFileVersion` | -| Extensions.Azure | `azurePackageVersion` | `azurePackagePreviewVersion` | `azureAssemblyFileVersion` | -| AKV Provider | `akvPackageVersion` | `akvPackagePreviewVersion` | `akvAssemblyFileVersion` | - -The pipeline resolves `effective*Version` variables at compile time based on the `isPreview` parameter. - -### 6.3 Release & Symbol Variables - -| Variable | Defined In | Purpose | -|----------|-----------|---------| -| `NuGetServiceConnection` | `libraries/common-variables.yml` | External NuGet service connection name for `NuGetCommand@2` push | -| `SymbolServer` | `libraries/common-variables.yml` (alias) | Alias for `$(SymbolsPublishServer)` — used by MDS `publish-symbols-step.yml` | -| `SymbolTokenUri` | `libraries/common-variables.yml` (alias) | Alias for `$(SymbolsPublishTokenUri)` — used by MDS `publish-symbols-step.yml` | - -### 6.4 Variable Groups - -| Group | Included In | Purpose | -|-------|------------|---------| -| `Release Variables` | `build-variables.yml` | Release-specific configuration | -| `Symbols publishing` | `build-variables.yml` | Symbol publishing credentials (`SymbolsAzureSubscription`, `SymbolsPublishServer`, `SymbolsPublishTokenUri`, `SymbolsUploadAccount`, `SymbolsPublishProjectName`) | -| `ESRP Federated Creds (AME)` | `common-variables.yml` | Federated identity for ESRP signing (`ESRPConnectedServiceName`, `ESRPClientId`, `AppRegistrationClientId`, `AppRegistrationTenantId`, `AuthAKVName`, `AuthSignCertName`) | - ---- - -## 7. Code Signing (ESRP) - -All packages are signed using **ESRP (Enterprise Security Release Pipeline)** with federated identity authentication. - -### Signing Flow (per job) - -#### csproj-based Extension Packages (Logging, Abstractions, Azure) -1. **Build DLLs only** — `build.proj` target (e.g., `BuildLogging`) compiles assemblies without creating NuGet packages -2. **ESRP DLL signing** — Assemblies are signed with Authenticode certificates via ESRP -3. **NuGet pack** — `build.proj` pack target (e.g., `PackLogging`) creates `.nupkg` from signed DLLs using `NoBuild=true` -4. **ESRP NuGet signing** — The `.nupkg` files are signed with NuGet certificates via ESRP - -This workflow ensures the NuGet package contains **signed DLLs** rather than signing the NuGet package around unsigned assemblies. - -#### nuspec-based Packages (SqlServer.Server, SqlClient, AKV Provider) -1. **Build + pack** — MSBuild creates both assemblies and NuGet packages -2. **ESRP DLL signing** — Assemblies are signed with Authenticode certificates via ESRP -3. **ESRP NuGet signing** — The `.nupkg` files are signed with NuGet certificates via ESRP - -### Credential Model - -- Extension packages (Logging, Abstractions, Azure, SqlServer.Server, SqlClient) use the primary ESRP credentials from the `ESRP Federated Creds (AME)` variable group (loaded via `common-variables.yml`) -- AKV Provider uses separate `Signing`-prefixed credential parameters that are passed explicitly to the `build-akv-official-job.yml` template -- All credentials are sourced from Azure Key Vault and federated identity — no secrets stored in pipeline YAML - ---- - -## 8. SDL & Compliance (OneBranch) - -Both pipelines use **OneBranch governed templates** for 1ES compliance. The SDL configuration differs between Official and Non-Official: - -| SDL Tool | Official | Non-Official | Purpose | -|----------|----------|--------------|---------| -| **TSA** | ✅ `enabled: true` | ❌ `enabled: false` | Uploads SDL results to TSA for downstream analysis | -| **ApiScan** | ✅ `enabled: true`, `break: true` | ✅ `enabled: true`, `break: true` | Scans APIs for compliance issues | -| **CodeQL** | ✅ (non-preview) | ✅ (non-preview) | Static analysis for security vulnerabilities | -| **SBOM** | ✅ (non-preview) | ✅ (non-preview) | Software Bill of Materials generation | -| **Policheck** | ✅ `break: true` | ✅ `break: true` | Scans for policy-violating content | -| **BinSkim** | ✅ (async, non-preview) | ✅ (async, non-preview) | Binary security analysis | -| **CredScan** | ✅ (async, non-preview) | ✅ (async, non-preview) | Credential leak detection | -| **Roslyn** | ✅ (async, non-preview) | ✅ (async, non-preview) | Roslyn-based security analyzers | -| **Armory** | ✅ `break: true` | ✅ `break: true` | Additional security scanning | - -### APIScan Configuration - -APIScan is configured at **both pipeline level and job level**: - -**Pipeline-level** (`globalSdl:apiscan:`): Sets default configuration inherited by all jobs. This is configured for MDS (Microsoft.Data.SqlClient) as the primary product. - -**Job-level** (`ob_sdl_apiscan_*` variables): Each build job overrides the pipeline defaults with package-specific settings: - -| Variable | Purpose | -|----------|---------| -| `ob_sdl_apiscan_enabled` | Enable/disable APIScan for this job (`true`) | -| `ob_sdl_apiscan_softwareFolder` | Path to signed DLLs for scanning | -| `ob_sdl_apiscan_symbolsFolder` | Path to PDBs for scanning | -| `ob_sdl_apiscan_softwarename` | Package name (e.g., `Microsoft.Data.SqlClient.Internal.Logging`) | -| `ob_sdl_apiscan_versionNumber` | Assembly file version | - -Each job copies its signed DLLs and PDBs to a package-specific folder under `$(Build.SourcesDirectory)/apiScan//` after ESRP DLL signing, ensuring APIScan analyzes the correct signed binaries for each package. - -> **PRC Compliance**: The Official pipeline hardcodes `OneBranch.Official.CrossPlat.yml` (not parameterized) to satisfy Production Readiness Check static verification requirements. - ---- - -## 9. Artifact Strategy - -- Each build job publishes its output as a **pipeline artifact** managed by OneBranch's `ob_outputDirectory` convention. -- Artifact names follow the OneBranch auto-generated pattern: `drop__` (e.g., `drop_build_dependent_build_package_SqlClient`). -- Downstream stages use `DownloadPipelineArtifact@2` to pull required packages into a local directory. -- A local NuGet source is configured at build time pointing to the downloaded artifacts directory so `dotnet restore` resolves internal dependencies. - ---- - -## 10. Trigger Configuration - -### Official Pipeline (`dotnet-sqlclient-official-pipeline.yml`) - -```yaml -trigger: - branches: - include: - - internal/main - paths: - include: - - .azuredevops - - .config - - doc - - eng/pipelines - - src - - tools - - azurepipelines-coverage.yml - - build.proj - - NuGet.config - -schedules: - - cron: '30 4 * * Mon' # Weekly Sunday 9:30 PM (UTC-7) - branches: { include: [internal/main] } - always: true - - cron: '30 3 * * Mon-Fri' # Weekday 8:30 PM (UTC-7) - branches: { include: [internal/main] } -``` - -- **CI trigger**: Runs on pushes to `internal/main` when relevant paths change -- **Scheduled**: Weekly full build (Sundays) + weekday builds (Mon–Fri) -- **No PR trigger**: Official pipeline should not run on PRs (separate PR pipelines exist) - -### Non-Official Pipeline (`dotnet-sqlclient-non-official-pipeline.yml`) - -```yaml -trigger: none -pr: none -``` - -- **Manual only**: Queued on-demand for validation/test builds - ---- - -## 11. Infrastructure - -| Concern | Implementation | -|---------|---------------| -| **Pipeline template** | OneBranch governed templates (`OneBranch.Pipelines/GovernedTemplates`) | -| **Build agents** | OneBranch-managed Windows containers (`WindowsHostVersion: 1ESWindows2022`) | -| **.NET SDK** | Pinned via `global.json` (with `useGlobalJson: true` in install steps) | -| **Code signing** | ESRP v2 with federated identity (Azure Key Vault backed) | -| **Symbol publishing** | Optional, controlled by `publishSymbols` parameter; uses `Symbols publishing` variable group (aliases `SymbolServer`/`SymbolTokenUri` defined in `common-variables.yml`) | - ---- - -## 12. Key Design Decisions - -1. **Single pipeline, multiple stages** — avoids managing 6 separate pipelines while keeping clear separation of concerns. -2. **Official + Non-Official variants** — hardcoded OneBranch templates (no parameterized `oneBranchType`) for PRC compliance; Non-Official variant allows manual validation builds. -3. **Parallel jobs within stages** — minimizes total wall-clock time; only waits where dependencies demand it. -4. **Pipeline artifacts over Universal Packages** — faster, ephemeral, scoped to the run; appropriate for build-time dependency resolution. -5. **ESRP-based code signing** — all DLLs and NuGet packages are signed in-pipeline using ESRP with federated identity; no secrets in YAML. -6. **Centralized version management** — all 6 package versions (GA + preview) defined once in `libraries/common-variables.yml`; `isPreview` toggle selects the active set. -7. **Dependency-aware stage ordering** — ensures packages are always built after their dependencies, guaranteeing consistent, reproducible builds. -8. **Validation in parallel with Stage 3** — MDS package validation runs alongside AKV Provider build (both depend on Stage 2), reducing total pipeline duration. -9. **Selective on-demand release** — 6 boolean parameters control which packages are published; the release stage is entirely skipped when none are selected, keeping normal CI builds unaffected. -10. **ADO Environment approval gate** — two environments: `NuGet-Production` (official, with configured approvals) and `NuGet-DryRun` (non-official, validation only). Both use `ob_release_environment` for OneBranch integration. -11. **Compile-time conditional publish jobs** — `${{ if eq(parameters.releaseXxx, true) }}` template expansion ensures unselected publish jobs are excluded entirely from the pipeline run (not just skipped at runtime). -12. **Mandatory dry-run for non-official** — the non-official variant hardcodes `dryRun: true` (no parameter), preventing accidental publication. The official variant defaults `releaseDryRun: true` for safety but allows override for actual releases. -13. **Selective build parameters** — `buildSqlClient`, `buildSqlServerServer`, and `buildAKVProvider` allow building subsets of packages, with dependency-aware conditionals ensuring Logging builds when either SqlClient or AKV is needed. +# OneBranch Pipeline Guidelines + +## Purpose + +Rules and conventions for editing the OneBranch Azure DevOps YAML pipelines that build, sign, package, and release six NuGet packages with interdependencies. + +## Pipeline Variants + +- `sqlclient-official.yml` — Official pipeline; uses `OneBranch.Official.CrossPlat.yml`; runs on a daily schedule at 23:00 UTC on `internal/release/7.1`, with no activity-based trigger (`pr: none`, `trigger: none`) +- `sqlclient-non-official.yml` — Non-Official pipeline; uses `OneBranch.NonOfficial.CrossPlat.yml`; manual only (`pr: none`, `trigger: none`) +- Both live under `eng/pipelines/onebranch/` and extend OneBranch governed templates +- Never parameterize the OneBranch template name — hardcode it per pipeline for PRC compliance +- Official pipeline must never be run on PRs or dev branches. + +## Package Dependency Order + +Respect this graph when modifying build stages: + +1. `Microsoft.SqlServer.Server` — no dependencies +2. `Microsoft.Data.SqlClient.Internal.Logging` — no dependencies +3. `Microsoft.Data.SqlClient.Extensions.Abstractions` — depends on Logging +4. `Microsoft.Data.SqlClient` — depends on Logging + Abstractions +5. `Microsoft.Data.SqlClient.Extensions.Azure` — depends on Abstractions + Logging +6. `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` — depends on SqlClient + Abstractions + Logging + +## Localization Validation + +The SqlClient build job runs `steps/validate-localization-step.yml` before building the driver. Validation always fails the build for missing or obsolete keys, empty localized values whose English value is non-empty, and untranslated resources. Approved identical translations are listed by culture and resource key in `.config/LocalizationValidationAllowlist.json`. + +## Build Stages + +Defined in `stages/build-stages.yml`. Four build stages plus package validation are ordered by dependency: + +- **`build_independent`** (Stage 1) — Logging and SqlServer.Server in parallel; no inter-package dependencies +- **`build_abstractions`** (Stage 2) — Abstractions; `dependsOn: build_independent`; downloads Logging artifact +- **`build_dependent`** (Stage 3) — SqlClient and Extensions.Azure in parallel; `dependsOn: build_abstractions`; downloads Abstractions + Logging artifacts +- **`build_addons`** (Stage 4) — AKV Provider; `dependsOn: build_dependent`; downloads SqlClient + Abstractions + Logging artifacts +- **`package_validation`** (Stage 5) — Validates every package produced by the run; `dependsOn` all four build stages plus `compute_versions` + +Each build job copies PDB files into `$(JOB_OUTPUT)/symbols/` so they are included in the auto-published pipeline artifact alongside the NuGet packages in `$(JOB_OUTPUT)/packages/`. + +Stage conditional rules: +- The SqlClient family (Logging, Abstractions, SqlClient, Azure, AKV Provider) is **always built** — Stages 2, 3, and 4 and the Logging job in Stage 1 are unconditional. There is no `buildSqlClient`/`buildAKVProvider` toggle. +- `buildSqlServer` is the only build toggle; it controls just the SqlServer.Server job in Stage 1. +- When `buildSqlServer` is true, SqlClient/AKV depend on the freshly-built SqlServer artifact (downloaded into the local feed). When false, they depend on the most recently published SqlServer package — a version-only dependency (no artifact download) restored from NuGet. + +## Job Templates + +- **`build-buildproj-job.yml`** — Shared build.proj-driven package job used for all shipped packages. Flow: build via `build.proj` → optional ESRP DLL signing → pack via `build.proj` → optional ESRP NuGet signing → copy outputs for APIScan/artifacts +- **`validate-packages-job.yml`** — Validates every package produced by the run. Downloads all package artifacts into one tree and validates them together, so `tools/PackageValidator` can apply its cross-package rules (the SqlClient family must share one version, and inter-package dependency ranges must agree); validating per package would silently skip those findings. Runs on Windows because Authenticode verification has no Linux equivalent +- **`publish-nuget-package-job.yml`** — Reusable release job using OneBranch `templateContext.type: releaseJob` with `inputs` for artifact download; pushes via `NuGetCommand@2` +- **`publish-symbols-job.yml`** — Reusable symbols job: downloads a build artifact, locates PDBs under `symbols/`, and invokes `publish-symbols-step.yml` + +When adding a new package to the OneBranch flow: +- Extend `build-buildproj-job.yml` inputs with the new package metadata and dependency artifacts +- Add or update the corresponding build/pack targets in `build.proj` +- Add version variables to `variables/common-variables.yml` +- Add artifact name variables to `variables/onebranch-variables.yml` + +## Package Validation Stage + +- Defined in `stages/build-stages.yml`; produces stage `package_validation` +- Consumes the package and file versions published by `compute_versions` and asserts the produced packages carry exactly those values, so nothing is re-derived +- All packages are validated together in one job so `tools/PackageValidator` can apply cross-package rules; the SqlServer artifact and its expectations are conditional on `buildSqlServer` +- Expectations use the validator's `[id=]value` form: the SqlClient family version is applied as a wildcard (proving the family agrees, and catching the case where all packages are consistently wrong), with `Microsoft.SqlServer.Server` as a per-id override +- When SqlServer is not built its expectations are **omitted entirely** rather than passed empty — the validator rejects an expectation with an empty value +- Gate categories are derived from `isOfficial`: `error`, `missing-symbols`, `dependency-inconsistency`, `delay-signed`, and `unsigned` always, plus `package-unsigned` on official runs only. The `error` severity covers only error-severity findings, so each warning/info category must be named explicitly — `missing-symbols`, `dependency-inconsistency`, and `delay-signed` are warnings, and `unsigned` and `package-unsigned` are info. Strong-name signing is unconditional in `build-buildproj-step.yml`, so the two strong-name categories gate everywhere; NuGet package signing is ESRP and official-only, so `package-unsigned` would fire on every non-official run +- The validator runs twice: once with `--json` and no gate so the report exists even for a failing run, then once human-readable with the gate so failures appear in the job log +- Signature verification (`dotnet nuget verify --all`, Authenticode) runs on official builds only, and verifies that signatures are *trusted* — PackageValidator reports only their presence, from metadata +- The release stage `dependsOn: package_validation`, so a package that fails validation is never published +- Step and job logic lives in `scripts/validate-packages.ps1`, `scripts/verify-package-signatures.ps1`, and `scripts/verify-assembly-signatures.ps1`, each with Pester tests under `scripts/tests/` + +## Symbols Publishing Stage + +- Defined in `stages/publish-symbols-stage.yml`; produces stage `publish_symbols` +- Entire stage excluded at compile time when `publishSymbols` is false +- The SqlClient family symbols are always published; the SqlServer.Server symbols job is conditional on `buildSqlServer` +- `dependsOn` covers all family build stages (always present), plus `build_independent` for SqlServer +- One job per package (`publish-symbols-job.yml`), each downloading its build artifact and publishing PDBs from `symbols/` +- Each package's PDBs are published separately with unique artifact names and version information +- Build jobs copy PDBs into `$(JOB_OUTPUT)/symbols/` so they are included in the auto-published artifact +- The `publish-symbols-step.yml` accepts a `symbolsFolder` parameter to point at the downloaded PDB location +- The publish step calls an extracted `publish-symbols.ps1` script with structured error handling and diagnostic logging +- Symbols publishing credentials come from the `Symbols Publishing` variable group +- In the official pipeline, symbol server destination follows `releaseToProduction`: Production when true, PPE when false +- Non-official pipeline always targets the PPE symbol server + +## Release Stage + +- Defined in `stages/release-stages.yml`; produces stage `release_production` (official) or `release_test` (non-official) via `stageNameSuffix` parameter +- Entire stage excluded at compile time when no release parameters are true +- `dependsOn` is conditional based on which release parameters are set +- `releaseToProduction` parameter controls NuGet target feed: + - `true` → service connection `ADO Nuget Org Connection` (NuGet Production) + - `false` → service connection `ADO Nuget Org Test Connection` (NuGet Test) +- Non-official pipeline always sets `releaseToProduction: false` +- Environment gating: + - Official: `ob_release_environment: Production`, `ob_deploymentjob_environment: NuGet-Production` + - Non-official: `ob_release_environment: Test`, `ob_deploymentjob_environment: NuGet-DryRun` +- Each publish job uses OneBranch deployment job syntax (`templateContext.type: releaseJob` with `inputs` for artifact download) + +## Parameters + +Build parameters: +- `debug` — enable debug output (default `false`) +- `isPreview` — use preview version numbers (default `false`) +- `publishSymbols` — publish symbols to servers (default `false`) +- `buildSqlServer` — build the Microsoft.SqlServer.Server package (default `true` in the non-official/nightly pipeline, `false` in the official pipeline). The SqlClient family is always built, so this is the only build toggle. It also drives the SqlServer dependency version the family uses (built/next vs published). Requesting `releaseSqlServer` without `buildSqlServer` fails template expansion. + +Release parameters (boolean, default `false`): +- `releaseSqlClient` — release the entire SqlClient family together (Logging, Abstractions, SqlClient, Azure, AKV Provider) at the shared version +- `releaseSqlServer` — release Microsoft.SqlServer.Server (versioned separately) + +Official-only parameter: +- `releaseToProduction` — controls both NuGet target feed and symbol server destination (default `false`): + - `true` → NuGet Production feed + Production symbol server + - `false` → NuGet Test feed + PPE symbol server + +When `isPreview` is true, pipeline resolves `effective*Version` variables to preview versions; otherwise GA versions. All versions defined in `variables/common-variables.yml`. + +## Variables and Versions + +- Variable chain: pipeline YAML → `variables/onebranch-variables.yml` → `variables/common-variables.yml` +- All package versions (GA, preview, assembly file) centralized in `variables/common-variables.yml` +- The `compute_versions` stage reads canonical versions from MSBuild and publishes effective package, + file-build, and APIScan registration versions for downstream stages +- Artifact name variables defined in `variables/onebranch-variables.yml` following `drop__` pattern +- `assemblyBuildNumber` derived from first segment of `Build.BuildNumber` only (16-bit limit) +- When adding a new package, add GA version, preview version, and assembly file version entries + +Variable groups: +- `Symbols Publishing` — symbol publishing credentials (in `onebranch-variables.yml`) +- `ESRP Federated Creds (AME)` — ESRP signing credentials (in `common-variables.yml`) + +## Code Signing (ESRP) + +- Uses ESRP v6 tasks (`EsrpMalwareScanning@6`, `EsrpCodeSigning@6`) with MSI/federated identity authentication +- Signing only runs when `isOfficial: true` — non-official pipelines skip ESRP steps +- The shared OneBranch job signs DLLs before packing and signs the resulting NuGet package afterward so the published package contains signed binaries +- DLL signing uses keyCode `CP-230012` (Authenticode); NuGet signing uses keyCode `CP-401405` +- All ESRP credentials come from variable groups — never hardcode secrets in YAML + +## SDL and Compliance + +- TSA: enabled only in official pipeline; disabled in non-official to avoid spurious alerts +- ApiScan: enabled in both; `break` follows the `breakOnSdlError` parameter +- Each package is registered with APIScan under its own name/version pair, so the `globalSdl.apiscan` blocks deliberately omit `softwareName`/`versionNumber`. `build-buildproj-job.yml` is the single place they are set, via `ob_sdl_apiscan_softwareName` (the package's `packageFullName`) and `ob_sdl_apiscan_versionNumber` (the `apiScanSoftwareVersion` parameter) +- `compute-versions.ps1` derives APIScan registration versions as major.minor from the effective canonical package versions and publishes them as stage outputs. A package name/version pair must still be registered with APIScan before releasing a new major.minor. Consume these as runtime `$(...)` references so values such as `1.0` remain strings rather than being coerced to numbers by template expressions +- Jobs that produce no assemblies (symbol publishing, signed-package validation, version computation) set `ob_sdl_apiscan_enabled: false` rather than reporting a name/version +- Each build job also sets `ob_sdl_apiscan_softwareFolder` and `ob_sdl_apiscan_symbolsFolder` to its per-package `apiScan//dlls` and `apiScan//pdbs` paths +- CodeQL, SBOM, Policheck (`break: true`): enabled in both pipelines +- SBOM package name/version are resolvable **only** from the pipeline's `globalSdl.sbom` block — OneBranch's artifact-publishing path reads `globalSdl.sbom.packageName`/`packageVersion` directly and has no per-job equivalent (the `templateContext.sdl.sbom` override only applies to the native 1ES Stages entry point, which this repo does not use). Because the pipeline produces six differently-named and independently-versioned packages, `globalSdl.sbom` indirects through the `$(sbomPackageName)` / `$(sbomPackageVersion)` variables, which each build job sets to its own `packageFullName` and computed `packageVersion`. Jobs that publish no packages (version computation, symbol publishing) set `ob_sdl_sbom_enabled: false` alongside their existing APIScan/BinSkim opt-outs, so the variables never need pipeline-level defaults +- asyncSdl `enabled: false` in both; individual sub-tools (CredScan, BinSkim, Armory, Roslyn) configured underneath +- Policheck exclusions: `$(REPO_ROOT)\.config\PolicheckExclusions.xml` +- CredScan suppressions: `$(REPO_ROOT)/.config/CredScanSuppressions.json` + +## Artifact Conventions + +- `ob_outputDirectory` set to `$(JOB_OUTPUT)` (= `$(REPO_ROOT)/output`) — OneBranch auto-publishes this directory +- Each published artifact uses subdirectories to separate file types: + - `assemblies/` — DLL assemblies for APIScan (preserving TFM folder structure) + - `packages/` — NuGet packages (`.nupkg`, `.snupkg`) + - `symbols/` — PDB symbol files (preserving TFM folder structure, shared by APIScan and symbol publishing) +- Artifact names follow `drop__` — defined in `variables/onebranch-variables.yml` +- Downstream jobs download artifacts via `DownloadPipelineArtifact@2` into `$(Build.SourcesDirectory)/packages` +- Downloaded packages serve as a local NuGet source for `dotnet restore` +- If stage or job names change, update artifact name variables in `onebranch-variables.yml` + +## Common Pitfalls + +- Do not use `PublishPipelineArtifacts` task — OneBranch auto-publishes from `ob_outputDirectory` +- Do not add `NuGetToolInstaller@1` in OneBranch containers — NuGet is pre-installed +- Variable templates are under `variables/` not `libraries/` +- Always test parameter changes in the non-official pipeline first +- When modifying stage names, update all `dependsOn` references and artifact name variables +- Release jobs must use `templateContext.type: releaseJob` with `inputs` for artifact download — deployment jobs do not auto-download artifacts diff --git a/.github/instructions/secrets.instructions.md b/.github/instructions/secrets.instructions.md new file mode 100644 index 0000000000..77af612bee --- /dev/null +++ b/.github/instructions/secrets.instructions.md @@ -0,0 +1,112 @@ +--- +applyTo: "**" +--- +# Secrets and Credential Handling + +This guide describes how to avoid committing secrets, how to write credential +placeholders that do **not** trip secret scanners (e.g. GitHub Advanced Security +/ 1ES push protection), and what to do when a push is blocked. + +## Golden Rules + +1. **Never commit a real secret** — passwords, connection strings with live + credentials, access keys, SAS tokens, client secrets, certificates/PFX + files, or bearer tokens. This applies to source, tests, docs, samples, + scripts, pipeline YAML, and config files. +2. **Never route a secret through tooling or the model.** When a value is truly + secret, have the user type it directly into their terminal or set it as an + environment variable. Do not paste it into files, prompts, or chat. +3. **Prefer indirection over literals.** Read credentials from environment + variables, a secret store (Azure Key Vault), user secrets, or CI secret + variables — not from committed text. +4. **Assume public.** This repository mirrors to public GitHub via autosync. + Anything committed is effectively public and permanent in history. + +## Approved Placeholder Formats + +When you need to show the *shape* of a connection string or credential in code, +docs, comments, or samples, use one of these placeholder styles. These are +recognized as non-secrets by the scanner: + +| Style | Example | Use for | +|-------|---------|---------| +| Angle brackets | `User ID=;Password=` | Docs, READMEs, comments, samples | +| Descriptive angle brackets | `Password=`, `User Id=` | Samples that name the value | +| Masked | `Password=********` or `Password=***` | Illustrative output / redaction | +| Env var expansion | `Password=${SA_PASSWORD}` (bash), `Password=$(SqlPwd)` (ADO) | Scripts and pipelines | +| Named token (docs prose) | `Password=` | Narrative documentation | + +### Do NOT use these placeholder styles + +- **Ellipsis values** — a `Password` (or `Pwd`) key whose value is a literal + ellipsis, especially when paired with a `User ID` key in the same connection + string. The ellipsis is treated as a credential value and **will** trip + `SEC101/037 SqlLegacyCredentials`. Use `` instead. +- **Realistic-looking fake secrets** — a `Password` key set to a value that + looks like a real password (mixed letters, digits, and symbols). Even fake + values that resemble real passwords can be flagged and set a bad example. +- **Bare word secrets** — a `Password` key set to a plain dictionary word + inside a connection-string literal. Prefer ``. + +> **NOTE**: Ironically, this document cannot show verbatim examples of the +> disallowed styles above — spelling out a `Password` key followed by an +> ellipsis or realistic-looking value would itself trip `SEC101/037` and block +> commits to this very file. That is exactly why the "don't" cases are described +> in prose rather than shown literally. + +### Full connection-string placeholder examples + +```text +Server=;Database=;User ID=;Password=;TrustServerCertificate=true +Server=tcp:.database.windows.net;Database=;Authentication=Active Directory Service Principal;User Id=;Password= +``` + +```bash +# Have the user set the value directly; never write the real value into a file: +export SNICLOSE_CONNSTR="Server=;User ID=;Password=" +``` + +## Reading Secrets at Runtime (preferred patterns) + +- **Environment variables**: read connection strings from an env var + (e.g. `SNICLOSE_CONNSTR`) so the password never lands in a committed file. +- **SecureString / SqlCredential**: use `SqlCredential` and `SecureString` + rather than embedding a password in the connection string. +- **Integrated auth**: prefer `Integrated Security=true` or an + `Authentication=ActiveDirectory*` mode where no password is stored. +- **CI/CD**: reference pipeline secret variables (`$(mySecret)`), never inline + literals in YAML. + +## When a Push Is Blocked by Secret Scanning + +Error shape: `VS403654:BypassableBlock ... push was rejected because it contains +one or more secrets` with a `SEC101/...` rule id and `commit`/`paths` details. + +1. **Locate it.** Inspect the exact committed blob: + `git show :` and go to the reported line/columns. +2. **Determine real vs. false positive.** + - *Real secret*: rotate/revoke it immediately, then remove it from the file. + If it is in the tip commit only, amend; if it is deeper in **unshared** + history, rewrite that history. Never rewrite commits already public. + - *False positive* (an ellipsis or other non-credential placeholder value): + reword to an approved placeholder (`Password=`) so future commits + don't recur. +3. **Already-public / mirrored commits.** If the flagged content lives in a + commit that is already on public GitHub (e.g. an autosync mirror replaying + `github/main`), you cannot scrub that specific commit without rewriting + shared/public history. For a confirmed false positive, **bypass** the block + via the 1ES/Advanced Security push-protection flow + (https://aka.ms/1esSecretScanning/PushProtectionBypassableBlock) with a clear + reason, or dismiss the alert as *False positive*. Then land the placeholder + reword going forward so new files don't trip the rule again. +4. **Never** disable secret scanning or use `--no-verify`-style bypasses to work + around a *real* secret. + +## Common Scanner Rules to Watch + +| Rule | Triggers on | +|------|-------------| +| `SEC101/037 SqlLegacyCredentials` | Connection strings pairing a `User ID` key with a `Password`/`Pwd` value | +| `SEC101/*` (general) | Cloud keys, SAS tokens, client secrets, PATs, bearer tokens | + +If in doubt, use an approved placeholder from the table above. diff --git a/.github/instructions/sqlclient-package-versions.instructions.md b/.github/instructions/sqlclient-package-versions.instructions.md new file mode 100644 index 0000000000..691367d9de --- /dev/null +++ b/.github/instructions/sqlclient-package-versions.instructions.md @@ -0,0 +1,189 @@ +--- +applyTo: "**/Versions.props,build.proj,eng/pipelines/**/*.yml" +--- +# SqlClient Package Version Resolution + +How package versions are determined across different build scenarios for the packages in this repository. + +## Package families + +The repository ships two independently-versioned units: + +- **The SqlClient family** — `Microsoft.Data.SqlClient` plus the packages that version in lockstep with + it: `Microsoft.Data.SqlClient.Internal.Logging`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, + `Microsoft.Data.SqlClient.Extensions.Azure`, and + `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`. **All family packages use the + SqlClient version numbers** — the same NuGet package version, file version, and assembly version — + and are always built and released together. +- **`Microsoft.SqlServer.Server`** — versioned and released on its own cadence. + +## Version Properties + +The SqlClient family version lives in `src/Microsoft.Data.SqlClient/Versions.props`, which is imported +for every project by `src/Directory.Build.props` (so the `SqlClient*` version properties are always +available). `Microsoft.SqlServer.Server` declares its own version in its own `Versions.props`. + +| Property | Applies to | Purpose | Example | +|----------|-----------|---------|---------| +| `SqlClientNextVersion` | SqlClient family | Version being developed; used for the next release | `7.1.0-preview1` | +| `SqlServerNextVersion` | SqlServer | Version being developed for SqlServer | `1.1.0-preview1` | +| `SqlServerPublishedVersion` | SqlServer | Last SqlServer version shipped to NuGet | `1.0.0` | + +The SqlClient family always ships its next version, so there is **no** family `PublishedVersion`. Only +`Microsoft.SqlServer.Server` keeps a published version (used when it is built as a SqlClient dependency +but not itself released). + +## Resolution Logic + +Each `Versions.props` uses a 3-tier `` block: + +| Priority | Condition | PackageVersion | FileVersion | +|----------|-----------|----------------|-------------| +| 1 | `PackageVersion` explicitly provided | Used as-is | Strip prerelease + append BuildNumber | +| 2 | `BuildNumber` provided (non-zero) | `NextVersion[-BuildSuffix]`, then `.BuildNumber` appended if that carries a prerelease tag | `NextVersion.Split('-')[0].BuildNumber` | +| 3 | Nothing provided | `NextVersion-dev` | `NextVersion.Split('-')[0].0` | + +For every family package, `` is `SqlClient` (e.g. `-p:SqlClientPackageVersion=...`); for +Microsoft.SqlServer.Server it is `SqlServer`. + +> **Two equivalent names for the same value.** `SqlClientPackageVersion` / `SqlServerPackageVersion` +> are the underlying project properties (read by `Versions.props`, `Directory.Packages.props`, the +> csproj files, and the nuspec). When building through `build.proj` — which the CI and OneBranch +> pipelines always do — pass the wrapper argument `-p:PackageVersionSqlClient=...` / +> `-p:PackageVersionSqlServer=...`; `build.proj` forwards it to the underlying +> `-p:SqlClientPackageVersion=...` / `-p:SqlServerPackageVersion=...`. Both set the same value — +> pipeline logs show `PackageVersion`, while project builds and props show `PackageVersion`. + +### Developer (local `dotnet build`) + +**Mode:** Project (default `ReferenceType=Project`) + +- No `BuildNumber`, no `BuildSuffix`, no `PackageVersion*` passed. +- Falls into Priority 3 (the `` branch). +- **Result:** `7.1.0-preview1-dev` / FileVersion `7.1.0.0` +- Dependencies are project references — no package versions needed for siblings. + +**Mode:** Package (`-p:ReferenceType=Package`) + +- Same version resolution for the package being built. +- Sibling dependencies are restored from local `packages/` feed (previously packed with `-dev` suffix). +- A developer would first `dotnet build build.proj -t:Pack` to produce local packages, then consume them. + +### PR Pipeline (non-official CI) + +**`buildSuffix`** set via core template parameter: +- `buildSuffix: 'pr'` (passed explicitly from PR pipeline) +- `BuildNumber` = `$(DayOfYear)$(Rev:rr)` (e.g. `15401` for day-of-year 154, run 01) + +**Mode:** Project (typical PR validation) + +- Versions computed in `compute-versions-ci-stage.yml` (runs `GetVersions*` targets with `-p:BuildSuffix=pr -p:BuildNumber=...`) +- Falls into Priority 2 with BuildSuffix present. +- **Result:** `7.1.0-preview1-pr.15401` / FileVersion `7.1.0.15401` +- Dependencies are project references — all packages built together in-tree. + +**Mode:** Package (PR package-ref validation) + +- Same version computation via compute-versions stage. +- Downstream stages define stage-level variables from compute-versions output using `$[ stageDependencies... ]`. +- Each build step passes the family wrapper argument `-p:PackageVersionSqlClient=` (and `-p:PackageVersionSqlServer=` where needed) to `build.proj`, which forwards it to the underlying `-p:SqlClientPackageVersion=` / `-p:SqlServerPackageVersion=` project property, hitting Priority 1. +- Sibling dependencies consumed from pipeline artifacts published by upstream stages. + +### CI Pipeline (non-official, triggered on merge) + +Same structure as PR but passes `buildSuffix: 'ci'` explicitly. + +- **Result:** `7.1.0-preview1-ci.15401` / FileVersion `7.1.0.15401` + +### OneBranch Pipeline (official) + +**Mode:** Always Package (`ReferenceType=Package`) + +Uses the full `compute-versions-stage.yml` machinery: + +#### Step A: Compute Versions (dedicated early stage) + +1. Runs the `GetVersionsSqlClient` and `GetVersionsSqlServer` MSBuild targets against `build.proj`. +2. Each target calls `dotnet build -getProperty:PackageVersion` and `-getProperty:FileVersion` with `BuildNumber` but **no BuildSuffix**. +3. Falls into Priority 2 without BuildSuffix. `NextVersion` already carries a prerelease tag on `main`, so the build number is appended (e.g. `7.1.0-preview1.26238.3`); on a release branch the stable `NextVersion` is used as-is (e.g. `7.1.0`). +4. `GetVersionsSqlServer` also extracts `SqlServerPublishedVersion` (the SqlClient family has no published version). + +#### Step B: Resolve Effective Versions + +- The **SqlClient family** always uses `SqlClientNextVersion`. +- **`Microsoft.SqlServer.Server`** uses its next or published version based on the + `buildSqlServer` boolean: + +| `buildSqlServer` | Effective SqlServer Version | Meaning | +|--------------------------|-----------------------------|---------| +| `True` | `SqlServerNextVersion` (e.g. `1.1.0-preview1`) | SqlServer is being released | +| `False` | `SqlServerPublishedVersion` (e.g. `1.0.0`) | Only built as a SqlClient dependency; use last-shipped | + +> **The default differs by pipeline.** The non-official (nightly) pipeline defaults `buildSqlServer: +> true`, exercising the "build SqlServer + local-feed dependency" flow; the official pipeline defaults +> `buildSqlServer: false`, exercising the "depend on the published SqlServer package" flow that +> matches actual release intent. Either can be overridden at queue time. Requesting `releaseSqlServer` +> without `buildSqlServer` fails template expansion up front. + +These are published as ADO output variables: `versions.SqlClientPackageVersion`, +`versions.SqlServerPackageVersion`, and their `*FileVersion` counterparts. + +#### Step C: Build Stages Consume Pre-computed Versions + +Each downstream build job receives: +- `packageVersion` parameter → passed as `-p:SqlClientPackageVersion=` (or `-p:SqlServerPackageVersion=` for SqlServer) +- Dependency versions → family dependencies use the shared `SqlClientPackageVersion`; the SqlServer dependency uses `SqlServerPackageVersion` + +Since an explicit `PackageVersion` is provided, Versions.props hits Priority 1 — uses the value verbatim. + +#### Package Version Shapes + +Both OneBranch pipelines use the human-readable run name `$(Year:YY)$(DayOfYear)$(Rev:.r)`, which the +compute stage receives as `Build.BuildNumber`. That run name drives the single supported package +version shape: it is appended after any prerelease suffix, reproducing the shape shipped by earlier +previews. + +- `1.2.3` stays `1.2.3` — non-preview releases are never stamped with a build number +- `1.2.3-preview1` becomes `1.2.3-preview1.`, e.g. `7.1.0-preview3.26238.3` +- The matching file version is `1.2.3.`, e.g. `7.1.0.26238` + +Note the asymmetry: the *package* version omits the build number for non-preview releases, but the +*file* version always carries one in its fourth component. This keeps every shipped assembly +date-encoded, while preserving the released package version customers expect. + +The file version's fourth component is only the *date* segment of the run name, because a four-part +file version cannot hold the full `.` value. Repeated runs on the same day therefore share +a file version even though their package versions differ. + +Only packages built in the current run are stamped. When `buildSqlServer` is `false`, the effective +SqlServer version remains `SqlServerPublishedVersion` so dependency restore continues to request the +package that actually exists on NuGet. + +#### Summary + +| Package | Version Source | Example | +|---------|----------------|---------| +| SqlClient family (always released together) | `SqlClientNextVersion` | `7.1.0-preview1` | +| SqlServer, being released | `SqlServerNextVersion` | `1.1.0-preview1` | +| SqlServer, dependency only | `SqlServerPublishedVersion` | `1.0.0` | + +## Key Architectural Difference + +| Scenario | Who computes versions | How dependencies get versions | +|----------|----------------------|-------------------------------| +| Developer | Versions.props inline (Priority 3) | Project references (no version needed) | +| PR/CI (Project) | `compute-versions-ci-stage` up-front | Project references (no version needed) | +| PR/CI (Package) | `compute-versions-ci-stage` up-front | Stage variables via `$[ stageDependencies... ]` → `-p:SqlClientPackageVersion=` / `-p:SqlServerPackageVersion=` | +| OneBranch | `compute-versions-stage` up-front | Explicit `-p:SqlClientPackageVersion=` / `-p:SqlServerPackageVersion=` from stage outputs | + +## Updating Versions + +After releasing the **SqlClient family**: +1. Update `SqlClientNextVersion` in `src/Microsoft.Data.SqlClient/Versions.props` to the next planned + version. (There is no family published version to update.) + +After releasing **`Microsoft.SqlServer.Server`**: +1. Update `SqlServerPublishedVersion` to the version just shipped. +2. Update `SqlServerNextVersion` to the next planned version. + +The SqlServer properties live in `src/Microsoft.SqlServer.Server/Versions.props`. diff --git a/.github/instructions/testing.instructions.md b/.github/instructions/testing.instructions.md index 4f651a7104..9f4032e070 100644 --- a/.github/instructions/testing.instructions.md +++ b/.github/instructions/testing.instructions.md @@ -9,11 +9,13 @@ applyTo: "**/tests/**,**/*Test*.cs" src/Microsoft.Data.SqlClient/tests/ ├── FunctionalTests/ # Tests without SQL Server dependency ├── ManualTests/ # Integration tests requiring SQL Server +├── PerformanceTests/ # Benchmark-style perf validation +├── StressTests/ # Long-running stress coverage ├── UnitTests/ # Unit tests with minimal dependencies └── tools/ └── Microsoft.Data.SqlClient.TestUtilities/ - ├── config.default.json # Template configuration - └── config.json # Local test configuration (git-ignored) + ├── config.default.jsonc # Template configuration + └── config.jsonc # Local test configuration (git-ignored) ``` ## Test Categories @@ -32,14 +34,14 @@ src/Microsoft.Data.SqlClient/tests/ ### Manual Tests (`ManualTests/`) - Full integration tests with SQL Server -- Require `config.json` setup +- Require `config.jsonc` setup - Test real database operations - Include Always Encrypted, Entra ID tests ## Test Configuration -### Setting Up `config.json` -Copy `config.default.json` to `config.json` and configure: +### Setting Up `config.jsonc` +Copy `config.default.jsonc` to `config.jsonc` and configure: ```json { @@ -67,7 +69,7 @@ Copy `config.default.json` to `config.json` and configure: ## Test Categories and Attributes ### Category Exclusions -Use `[Trait("Category", "...")]` (xUnit, used in both ManualTests and UnitTests) to mark test categories and exclusions: +Use `[Trait("category", "...")]` (xUnit, used in both ManualTests and UnitTests) to mark test categories and exclusions: | Category | Excluded On | Description | |----------|-------------|-------------| @@ -80,7 +82,7 @@ Use `[Trait("Category", "...")]` (xUnit, used in both ManualTests and UnitTests) | `flaky` | All platforms (quarantine) | Intermittently failing tests (see Quarantine Zone below) | ### Flaky Test Quarantine Zone -Tests that intermittently fail are quarantined with `[Trait("Category", "flaky")]`. Quarantined tests: +Tests that intermittently fail are quarantined with `[Trait("category", "flaky")]`. Quarantined tests: - Are **excluded** from regular test runs by the default filter: `category!=failing&category!=flaky` - Run in **separate quarantine pipeline steps** to track their status without blocking CI - Do **not** collect code coverage @@ -94,35 +96,35 @@ Tests that intermittently fail are quarantined with `[Trait("Category", "flaky") **How to quarantine:** ```csharp // For unit tests (xUnit Trait) -[Trait("Category", "flaky")] +[Trait("category", "flaky")] public class FlakyConnectionTests { ... } // For individual test methods -[Trait("Category", "flaky")] +[Trait("category", "flaky")] [ConditionalFact(...)] public async Task OpenAsync_TimingDependent_MayFail() { ... } ``` -**How to un-quarantine:** Remove the `[Trait("Category", "flaky")]` attribute once the root cause is fixed and the test passes consistently. +**How to un-quarantine:** Remove the `[Trait("category", "flaky")]` attribute once the root cause is fixed and the test passes consistently. ### Test Timeout Enforcement All test runs use `--blame-hang-timeout 10m` to kill tests that hang for more than 10 minutes. This is configured in `build.proj` and applied to all test targets. If a test is expected to run longer than 10 minutes, it must be restructured or split. ### Test Filter Configuration -The default test filter is defined in `build.proj`: +The default test filter is defined in `build.proj` via `TestFilters`: ```xml -category!=failing&category!=flaky +category!=failing&category!=flaky&category!=interactive ``` -This can be overridden via MSBuild property: `msbuild -p:FilterStatement="your_filter"`. +This can be overridden via build property: `dotnet build build.proj -t:TestSqlClientUnit -p:TestFilters="your_filter"`. ### Test Attributes ```csharp // Platform-specific exclusion -[Trait("Category", "nonlinuxtests")] +[Trait("category", "nonlinuxtests")] public void TestWindowsSpecificFeature() { ... } // Skip on .NET Framework -[Trait("Category", "nonnetfxtests")] +[Trait("category", "nonnetfxtests")] public void TestNetCoreOnlyFeature() { ... } // Conditional skip based on test configuration @@ -130,26 +132,26 @@ public void TestNetCoreOnlyFeature() { ... } public void TestRequiresDatabase() { ... } // Quarantined flaky test -[Trait("Category", "flaky")] +[Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] public void TestIntermittentlyFails() { ... } ``` ## Running Tests -### Using MSBuild (Recommended) +### Using `build.proj` targets (Recommended) ```bash # Build and run all unit tests -msbuild -t:RunUnitTests +dotnet build build.proj -t:TestSqlClientUnit # Run functional tests only -msbuild -t:RunFunctionalTests +dotnet build build.proj -t:TestSqlClientFunctional # Run manual tests for specific framework -msbuild -t:RunManualTests -p:TF=net8.0 +dotnet build build.proj -t:TestSqlClientManual -p:TestFramework=net8.0 # Run specific test set -msbuild -t:RunManualTests -p:TestSet=1 +dotnet build build.proj -t:TestSqlClientManual -p:TestSet=1 ``` ### Using dotnet CLI @@ -159,7 +161,7 @@ dotnet test "src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft.Data.SqlClie -p:Configuration=Release # Functional tests with filter (excludes failing, flaky, and interactive tests) -dotnet test "src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.Tests.csproj" \ +dotnet test "src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj" \ --filter "category!=failing&category!=flaky&category!=interactive" # Run ONLY quarantined flaky tests (for investigation) @@ -171,6 +173,45 @@ dotnet test ... --filter "FullyQualifiedName=Namespace.ClassName.MethodName" ## Writing Tests +### Test Documentation Requirements + +To keep tests maintainable for contributors and AI agents, test intent must be documented at +class and method level. + +#### Required XML Documentation +- Add XML `` comments to every test class. +- Add XML `` comments to every test method (`[Fact]`, `[Theory]`, conditional variants). +- For helper methods used by tests, add XML `` comments and XML `` / `` + where applicable. +- For fixture and collection types, add XML `` comments describing why the fixture exists + (for example, serialization of console-mutating tests). + +#### What the Comments Must Explain +- The behavior/contract being tested (not just restating the method name). +- Why the scenario matters (for example: regression guard, parsing contract, sync/async parity, + isolation requirement). +- For helper methods, what side effects occur (for example console redirection, file system + copying, process execution) and why they are needed. + +#### Style Guidance +- Keep comments concise and factual. +- Prefer behavior-focused wording over implementation trivia. +- Avoid comments that merely repeat obvious code. +- Use inline comments inside test methods only for non-obvious setup/act/assert details. + +#### Example +```csharp +/// +/// Ensures malformed connection strings return a non-zero exit code and emit a parse error +/// without verbose exception details. +/// +[Fact] +public void AppRunWithMalformedConnectionStringReturnsOneAndWritesParseError() +{ + // Arrange / Act / Assert +} +``` + ### Test Structure ```csharp public class FeatureNameTests @@ -319,11 +360,43 @@ Extended assertions for SqlClient: AssertExtensions.ThrowsContains(() => action(), "expected message"); ``` +### RAII Database Object Classes +When writing manual integration tests that require transient database objects, use the RAII classes from `Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects` instead of manually writing `try/finally` blocks with DDL `DROP`/`CREATE` statements. + +**Available classes:** + +| Class | SQL generated | Example definition argument | +|-------|--------------|----------------------------| +| `Table` | `CREATE TABLE {Name} {definition}` | `"(Id INT, Value NVARCHAR(100))"` | +| `StoredProcedure` | `CREATE PROCEDURE {Name} {definition}` | `"AS BEGIN SELECT 1 END"` | +| `UserDefinedType` | `CREATE TYPE [dbo].{Name} AS {definition}` | `"TABLE (f1 INT)"` | + +Each class generates a unique object name from the given prefix (incorporating a timestamp-based GUID, username, and machine name), creates the object on construction (requiring the connection to already be open), and drops it when disposed. The generated name is available via the `.Name` property. + +**Pattern:** +```csharp +using SqlConnection conn = new(DataTestUtility.TCPConnectionString); +conn.Open(); + +using Table testTable = new(conn, "MyTable", "(Id INT, Name NVARCHAR(100))"); +using StoredProcedure proc = new(conn, "MyProc", $"AS BEGIN SELECT * FROM {testTable.Name} END"); + +using SqlCommand cmd = conn.CreateCommand(); +cmd.CommandText = proc.Name; +cmd.CommandType = CommandType.StoredProcedure; +// ... objects are automatically dropped when the scope ends +``` + +**Rules:** +- Open the connection **before** constructing any database object (the constructor executes DDL immediately) +- When objects depend on each other (e.g., a stored procedure that references a table), declare the dependent object **last** so it is disposed first — `using` declarations are disposed in reverse order +- Use the `.Name` property directly wherever you need to reference the object in SQL; for `UserDefinedType` this already includes the `[dbo].` schema prefix, making it suitable for use as a TVP `TypeName` + ## Code Coverage ### Running with Coverage ```bash -msbuild -t:RunTests -p:CollectCoverage=true +dotnet build build.proj -t:TestSqlClientUnit -p:TestCodeCoverage=true ``` ### Coverage Targets @@ -333,16 +406,11 @@ msbuild -t:RunTests -p:CollectCoverage=true ## Debugging Tests -### Visual Studio +### IDE 1. Set breakpoints in test code -2. Right-click test → Debug Test +2. Right-click test → Debug Test (or use CodeLens "Debug Test" link) 3. Use Test Explorer for navigation -### VS Code -1. Configure C# extension -2. Use CodeLens "Debug Test" link -3. Attach to test process - ### Command Line ```bash # Enable verbose output diff --git a/.github/plans/apicompat-ref-assembly-validation.md b/.github/plans/apicompat-ref-assembly-validation.md index b968dbf678..3ae4f0a46d 100644 --- a/.github/plans/apicompat-ref-assembly-validation.md +++ b/.github/plans/apicompat-ref-assembly-validation.md @@ -9,7 +9,7 @@ The comparison uses `Microsoft.DotNet.ApiCompat.Tool` in **strict mode**, which ## Usage ``` -dotnet msbuild build.proj /t:CompareRefAssemblies /p:BaselinePackageVersion=6.1.4 +dotnet build build.proj /t:CompareMdsRefAssemblies /p:BaselinePackageVersion=6.1.4 ``` - `BaselinePackageVersion` is **required** (no default). The user must specify which published package to compare against. @@ -22,7 +22,7 @@ dotnet msbuild build.proj /t:CompareRefAssemblies /p:BaselinePackageVersion=6.1. Add an entry for version `9.0.200` alongside the existing `dotnet-coverage` entry. This enables `dotnet apicompat` after `dotnet tool restore`. -### 2. Create `tools/targets/CompareRefAssemblies.targets` +### 2. Create `tools/targets/CompareMdsRefAssemblies.targets` A single new file containing all properties, items, and targets (steps 3–11 below). Follows the naming convention of existing files like `GenerateMdsPackage.targets`. @@ -85,7 +85,7 @@ Runs ``. - Preceded by `` labelling each comparison - Uses item metadata to map `net462` to `$(LegacyNetFxRefDir)` and others to `$(LegacyNetCoreRefDir)` -### 11. `CompareRefAssemblies` target (public entry point) +### 11. `CompareMdsRefAssemblies` target (public entry point) - Declared with `DependsOnTargets="_RunRefApiCompat"` - Emits a final `` summarizing completion @@ -95,12 +95,12 @@ Runs ``. Add one line after the existing `.targets` imports (after line 7): ```xml - + ``` ## Design Decisions -- **Single new file** at `tools/targets/CompareRefAssemblies.targets` — only one `` line added to `build.proj`. +- **Single new file** at `tools/targets/CompareMdsRefAssemblies.targets` — only one `` line added to `build.proj`. - **Internal targets prefixed with `_`** to signal they're not intended to be called directly. - **Strict mode** ensures API additions are also flagged — important for detecting accidental public surface changes during file reorganization. - **`ContinueOnError="ErrorAndContinue"`** on each apicompat `Exec` so all 8 comparisons run and all differences are reported together. @@ -111,7 +111,7 @@ Add one line after the existing `.targets` imports (after line 7): ## Verification ``` -dotnet msbuild build.proj /t:CompareRefAssemblies /p:BaselinePackageVersion=6.1.4 +dotnet build build.proj /t:CompareMdsRefAssemblies /p:BaselinePackageVersion=6.1.4 ``` - Downloads 6.1.4 nupkg, builds both ref project variants, runs 8 comparisons (4 TFMs × 2 variants). diff --git a/.github/prompts/ado-work-item-agent.prompt.md b/.github/prompts/ado-work-item-agent.prompt.md index 6eb6319ef6..e94d667bac 100644 --- a/.github/prompts/ado-work-item-agent.prompt.md +++ b/.github/prompts/ado-work-item-agent.prompt.md @@ -23,7 +23,7 @@ Perform the following steps to address the work item. Think step-by-step. - Locate the relevant code in `src/` or `tests/`. ### 2. Planning and Branching -- Propose a descriptive branch name following the pattern `dev/username/branch-name` (e.g., `dev/jdoe/fix-connection-pool`). +- Propose a descriptive branch name following the repository rule `dev/automation/` (e.g., `dev/automation/fix-connection-pool`). - Identify any dependencies or potential breaking changes. ### 3. Implementation @@ -33,13 +33,13 @@ Perform the following steps to address the work item. Think step-by-step. ### 4. Testing and Verification - **Mandatory**: All changes must be tested. -- Create new unit tests in `tests/UnitTests` or functional tests in `tests/FunctionalTests` as appropriate. +- Create new unit tests in `src/Microsoft.Data.SqlClient/tests/UnitTests` or functional tests in `src/Microsoft.Data.SqlClient/tests/FunctionalTests` as appropriate. - Verify that the tests pass. ### 5. Documentation and Finalization - If public APIs are modified, update the documentation in `doc/`. - Provide a clear summary of changes for the Pull Request. -- Suggest an entry for [CHANGELOG.md](CHANGELOG.md) if the change is significant. +- Suggest a release-note entry under `release-notes/` or in the PR description if the change is significant; do not edit `CHANGELOG.md` directly. ## Input **Work Item ID**: ${input:workItemId} diff --git a/.github/prompts/audit-variable-groups.prompt.md b/.github/prompts/audit-variable-groups.prompt.md new file mode 100644 index 0000000000..075e12d99a --- /dev/null +++ b/.github/prompts/audit-variable-groups.prompt.md @@ -0,0 +1,164 @@ +--- +name: audit-variable-groups +description: Audit Azure DevOps variable groups by searching repos/branches for usage and updating descriptions accordingly. +argument-hint: +tools: ['execute/runInTerminal', 'execute/getTerminalOutput', 'edit/createFile', 'read/readFile'] +--- + +Audit Azure DevOps variable groups in the **sqlclientdrivers** organization, **ADO.Net** project. Use the `az` CLI where possible; fall back to direct REST API calls where `az` doesn't provide sufficient coverage (e.g., repo file scanning, AzureKeyVault group updates). + +## Safety constraints + +- **Read-only by default.** Steps 1–4 are purely read-only (listing, searching, summarizing). Do NOT issue any write operations (`PUT`, `PATCH`, `POST`, `az pipelines variable-group update`, or any command that modifies state) until the user has explicitly approved changes in step 5. +- **No implicit approval.** Silence, ambiguous replies, or partial acknowledgements do NOT count as approval. You must receive an unambiguous "go" (or equivalent affirmative) from the user before proceeding to step 6. +- **Scope lock.** Only update variable group descriptions. Never delete variable groups, modify variables within a group, or change any pipeline definitions. +- **Dry-run first.** When presenting the summary in step 4, show the exact before/after description text for every group that would be modified so the user can verify the changes. +- **Abort on doubt.** If any step produces unexpected errors, ambiguous results, or data that contradicts expectations, stop and ask the user for guidance rather than proceeding. + +## Inputs + +If the user provided arguments, parse `${input:scope}` for overrides — it may contain specific repos, branches, or variable group names to scope the audit. Apply any recognized values as overrides to the defaults below; ignore unrecognized tokens. + +The user may override any of the following defaults: + +- **Organization**: `https://sqlclientdrivers.visualstudio.com` +- **Project**: `ADO.Net` +- **Repos & branches to search**: + - `dotnet-sqlclient`: `internal/main`, `internal/release/7.0`, `internal/release/6.1` + - `Microsoft.Data.SqlClient`: `ConfigFuzz` + - `Microsoft.Data.SqlClient.Ctaip`: `certAuth` + - `Microsoft.Data.SqlClient.sni`: `master`, `release/6.0` +- **Unused marker text**: `UNUSED - WILL BE DELETED SHORTLY` + +## Workflow + +### 1. List all variable groups + +``` +az pipelines variable-group list --org --project -o json +``` + +- Save the output. +- Identify which groups are already marked with the unused marker text and which are active. +- Use **fuzzy matching** when detecting the unused marker: check for the presence of both "unused" and "delete" (case-insensitive) in the description, since the actual marker text may vary (e.g. a person's name inserted before "WILL DELETE"). +- In later steps, ignore the current group description when determining usage to avoid biasing the search results. + +### 2. Search repos for variable group references + +For each repo/branch combination, use the Azure DevOps REST API (Items endpoint) to: + +1. List all files recursively in the repo at the given branch. +2. Filter to `.yml` and `.yaml` files. +3. Fetch the content of each YAML file. +4. Search for each variable group name using **exact-match** patterns that prevent prefix false positives (e.g., searching for `Foo` must not match `FooBar`). Match against these forms, ensuring the name is delimited by quotes or end-of-value (whitespace/newline/comment): + - `group: ''` (single-quoted — name bounded by quotes) + - `group: ""` (double-quoted — name bounded by quotes) + - `group: ` followed by end-of-line, whitespace, or `#` (unquoted — no trailing alphanumeric characters) + +Use a Bearer token from `az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query accessToken -o tsv`. + +REST API endpoints: +- **List items**: `{org}/{project}/_apis/git/repositories/{repo}/items?recursionLevel=Full&versionDescriptor.version={branch}&versionDescriptor.versionType=branch&api-version=7.1` +- **Get file content**: Same endpoint with `path={URL-encoded filePath}&$format=text` (URL-encode the `path` value; use `$format=text` to retrieve raw file content instead of JSON metadata) + +Avoid cloning repos. Only use the REST API to fetch file listings and content. + +**Performance & rate-limiting guidance**: +- Filter the file listing to paths likely to contain pipelines (e.g., `eng/`, `pipelines/`, or root-level YAML files) before fetching content, to reduce API calls. +- Add a short delay (e.g., 200ms) between file-content fetches to avoid hitting Azure DevOps rate limits. +- If a `429 Too Many Requests` or `503` response is received, back off exponentially (1s, 2s, 4s, …) and retry up to 3 times before logging a warning and moving on. + +### 3. Search Classic pipelines for variable group references + +Query all **enabled** Classic build and release pipeline definitions for variable group usage. + +#### Classic Build pipelines + +``` +GET {org}/{project}/_apis/build/definitions?api-version=7.1 +``` + +- Page through all results (`$top` / `continuationToken` if needed). +- Keep only definitions where `queueStatus` is **`enabled`**. +- For each enabled definition, fetch its full JSON: + ``` + GET {org}/{project}/_apis/build/definitions/{id}?api-version=7.1 + ``` +- Inspect the `variableGroups` array; each element has an `id` that maps to a variable group ID. + +#### Classic Release pipelines + +The Release API lives on the `vsrm.` sub-domain, but direct REST calls to that sub-domain may fail with SSL certificate errors for `.visualstudio.com` organizations. Use `az devops invoke` instead: + +``` +az devops invoke --area release --resource definitions \ + --org --route-parameters project= \ + --query-parameters '$expand=environments' '$top=200' \ + -o json +``` + +- Page through results using `continuation_token` if present in the response. +- Exclude definitions where `isDeleted` is `true`. +- Each definition can reference variable groups at two levels: + - **Definition level**: `variableGroups` array on the root object. + - **Stage/environment level**: each element in `environments` has its own `variableGroups` array. +- Collect all referenced variable group IDs from both levels. + +#### Recording results + +For every variable group ID found, record: +- The pipeline **name** and **type** (Build / Release). +- The stage name (for release-environment-level references). + +Merge these results with the repo/branch search results from step 2 so the summary in the next step covers both YAML and Classic usage. + +### 4. Summarize findings + +Present a clear summary table to the user **before making any changes**. The summary must include: + +- **Used groups**: group name, ID, which repos/branches and/or Classic pipelines reference it, and the proposed new description. +- **Unused groups**: group name, ID, current description, and confirmation it will be marked with the unused marker. +- **Surprise findings**: any group already marked unused that is actually still referenced (these should be un-marked). +- **No-change groups**: groups already marked unused and confirmed unused. + +### 5. Prompt for go/no-go + +**This is a mandatory gate — do NOT skip or auto-approve.** + +Ask the user to confirm before applying any changes and require an explicit selection from the options below (do not infer approval from ambiguous replies). Offer options: +- **Apply all** — apply every proposed change from step 4 +- **Apply subset** — let the user specify which groups to update (by name or ID) +- **Export script** — write all update commands to a shell script file for the user to inspect and run manually. Do NOT execute any updates. Save the script to a path the user specifies (default: `./audit-variable-group-updates.sh`). The script must include the full `az` CLI commands and REST API `curl` fallbacks (for AzureKeyVault-type groups) with comments identifying each group by name and ID. Mark the file executable. +- **Abort** — make no changes at all + +Do NOT proceed to step 6 unless the user selects "Apply all" or "Apply subset" and, for the subset case, clearly identifies which groups to update. If the user selects "Export script", generate the file and stop — do NOT execute step 6. If the user says "abort" or does not respond, stop here. + +### 6. Apply description updates + +**Prerequisites**: Step 5 must have completed with explicit user approval. If you have not received approval, do NOT execute this step. + +For each group that needs updating, try: + +``` +az pipelines variable-group update --group-id --description "" --org --project --detect false +``` + +**Fallback for AzureKeyVault-type groups**: The `az pipelines variable-group update` command may fail with 500 errors on groups whose `type` is `AzureKeyVault` (it tries to refresh the vault connection). When this happens, fall back to the REST API: + +1. `GET {org}/{project}/_apis/distributedtask/variablegroups/{id}?api-version=7.1` +2. Update **both** the top-level `description` field **and** every entry in `variableGroupProjectReferences[].description` in the JSON. +3. `PUT` the modified JSON back to the same URL with `Content-Type: application/json`. + +**Description rules**: +- **Used groups**: Prepend `[Used by: : , ; : ; Classic/: ] ` to the existing description (after stripping any previous `[Used by: ...]` prefix). `` is `Build` or `Release`. +- **Unused groups**: Set description to the unused marker text. +- **Incorrectly marked unused**: Replace the unused marker with `[Used by: ...]`. +- **Already correct**: Skip groups whose description would not change. + +Report the outcome of each update (success/failure) and a final tally. + +## Error handling + +- If a repo or branch does not exist or returns an error, log a warning and continue with the remaining repos/branches. +- If a variable group update fails, log the error and continue with the remaining updates. +- At the end, report any failures so the user can address them manually. diff --git a/.github/prompts/code-review.prompt.md b/.github/prompts/code-review.prompt.md index 6196c32293..3a64abec0b 100644 --- a/.github/prompts/code-review.prompt.md +++ b/.github/prompts/code-review.prompt.md @@ -1,18 +1,20 @@ --- name: code-review -description: AI-assisted code review for a pull request in Microsoft.Data.SqlClient. +description: AI-assisted code review for a pull request or branch in Microsoft.Data.SqlClient. argument-hint: agent: agent -tools: ['github/search_issues', 'read/readFile', 'codebase/search'] +tools: ['github/search_issues', 'github/pull_request_read', 'github/get_file_contents', 'github/run_secret_scanning', 'read/readFile', 'search'] --- -Review the pull request "${input:pr}" in `dotnet/SqlClient`. +Review the changes in "${input:target}" for `dotnet/SqlClient`. + +The target may be either a **PR number** (e.g., `4106`) or a **branch name** (e.g., `dev/user/my-feature`). Determine which by checking whether the value is purely numeric. Follow this structured review process: ## 1. Understand the Change -- Fetch the PR details: title, description, linked issue(s), and diff. - Read the PR description to understand the intent and scope of the change. +- Check for linked issues referenced in the description (e.g., `Fixes #...`). - Check which files are modified and categorize them: - **Source code** (`src/Microsoft.Data.SqlClient/src/`) — the main review focus - **Tests** (`tests/`) — verify coverage diff --git a/.github/prompts/fix-bug.prompt.md b/.github/prompts/fix-bug.prompt.md index 47b143f31f..ade780708d 100644 --- a/.github/prompts/fix-bug.prompt.md +++ b/.github/prompts/fix-bug.prompt.md @@ -25,9 +25,9 @@ Follow this workflow step-by-step: ## 3. Write a Failing Test - Create a test that reproduces the bug BEFORE implementing the fix. - Choose the correct test project: - - `tests/UnitTests/` — for isolated logic tests (no SQL Server needed) - - `tests/FunctionalTests/` — for API behavior tests (no SQL Server needed) - - `tests/ManualTests/` — for integration tests (requires SQL Server) + - `src/Microsoft.Data.SqlClient/tests/UnitTests/` — for isolated logic tests (no SQL Server needed) + - `src/Microsoft.Data.SqlClient/tests/FunctionalTests/` — for API behavior tests (no SQL Server needed) + - `src/Microsoft.Data.SqlClient/tests/ManualTests/` — for integration tests (requires SQL Server) - Follow existing naming conventions: `{ClassName}Tests.cs` with methods named `{MethodName}_{Scenario}_{ExpectedResult}`. - If the bug is platform-specific, add appropriate `[ConditionalFact]` or `[ConditionalTheory]` attributes with `[PlatformSpecific]`. - **Cover both sync and async code paths** if the affected API has both variants (e.g., `Open`/`OpenAsync`, `ExecuteReader`/`ExecuteReaderAsync`). Sync and async paths often have different internal implementations and a bug may manifest in only one. diff --git a/.github/prompts/generate-doc-comments.prompt.md b/.github/prompts/generate-doc-comments.prompt.md index 33ced936de..9361d3c7ad 100644 --- a/.github/prompts/generate-doc-comments.prompt.md +++ b/.github/prompts/generate-doc-comments.prompt.md @@ -1,5 +1,5 @@ --- -name: doc-comments +name: generate-doc-comments description: Generate XML documentation comments for C# code following .NET best practices. argument-hint: agent: agent diff --git a/.github/prompts/generate-prompt.prompt.md b/.github/prompts/generate-prompt.prompt.md index a4aa8cd40f..4db6c577c9 100644 --- a/.github/prompts/generate-prompt.prompt.md +++ b/.github/prompts/generate-prompt.prompt.md @@ -2,6 +2,7 @@ name: generate-prompt description: Generates high-quality VS Code Copilot prompt files (.prompt.md) based on user descriptions, leveraging available skills. argument-hint: Describe the prompt you want to create (e.g., "A prompt to generate unit tests for C#") +tools: [read, edit, search, todo] --- You are an expert AI prompt developer specialized in creating **Visual Studio Code Copilot Prompt Files (`.prompt.md`)**. @@ -34,6 +35,7 @@ Before generating the prompt, review the available skills in the `.github/skills * `name`: A concise, kebab-case name for the prompt. * `description`: A clear, short description of what the prompt does. * `argument-hint`: (Optional) A hint for what arguments the user can provide when using the prompt. + * `tools`: (Recommended) A list of tool identifiers the prompt is allowed to use. See the **Tool Scoping** section below. * **Body Structure**: * **Role**: Define the AI's persona (e.g., "You are an expert C# developer..."). * **Context**: Include specific context instructions or references. @@ -50,12 +52,70 @@ Before generating the prompt, review the available skills in the `.github/skills * Use `${input:variableName}` for user inputs (e.g., `${input:methodName}`). * Use built-in variables like `${selection}`, `${file}`, or `${workspaceFolder}` where appropriate context is needed. -6. **Best Practices**: +6. **Scope Tools**: Restrict the tools available to each prompt using the `tools` frontmatter field. See the **Tool Scoping** section below for detailed guidance. + +7. **Best Practices**: * Be specific and explicit. * Encourage chain-of-thought reasoning if the task is complex. * Reference workspace files using Markdown links `[path/to/file.cs](path/to/file.cs)` only if they are static and necessary for *all* invocations of this prompt. * Prefer referencing skills over duplicating instructions that already exist in skills. +## Tool Scoping + +Every generated prompt **should** include a `tools` list in its YAML frontmatter. Scoping tools keeps the model focused by limiting it to approved, known-effective tools for the task. Without tool scoping, the model may invoke irrelevant tools, waste context, or produce unpredictable results. + +### Why scope tools? +- **Focus**: Fewer tools means the model spends less reasoning on tool selection and more on the task. +- **Reliability**: Restricting to tested tools avoids unexpected side effects (e.g., a read-only review prompt shouldn't have edit tools). +- **Safety**: Prevents prompts from accidentally running terminal commands or making file changes when they shouldn't. + +### How to choose tools +Apply the **principle of least privilege** — include only the tools the prompt actually needs: + +| Prompt type | Recommended tools | +|---|---| +| **Read-only analysis** (review, triage, explain) | `read/readFile`, `search/codebase`, `search/textSearch` | +| **Code editing** (bug fix, feature, refactor) | `edit/editFiles`, `edit/createFile`, `read/readFile`, `search/codebase` | +| **Needs terminal** (build, test, scripts) | All of the above + `execute/runInTerminal`, `execute/getTerminalOutput` | +| **Needs GitHub data** (triage, release notes) | All of the above + `github/search_issues` or other GitHub tools | +| **Needs web content** (docs lookup) | `web/fetch` | + +### Available built-in tool identifiers + +You can specify individual tools or tool sets (which include all tools in that group). + +**Tool sets** (use these to include all tools in a category): +- `edit` — File creation and editing tools +- `read` — File and notebook reading tools +- `search` — Codebase, text, and file search tools +- `execute` — Terminal, task, and notebook execution tools +- `web` — Web content fetching tools + +**Commonly used individual tools:** + +| Tool identifier | Purpose | +|---|---| +| `edit/editFiles` | Apply edits to existing files | +| `edit/createFile` | Create a new file | +| `read/readFile` | Read file contents | +| `read/problems` | Get workspace problems/diagnostics | +| `search/codebase` | Semantic code search | +| `search/textSearch` | Text/regex search in files | +| `search/fileSearch` | Search for files by glob pattern | +| `search/listDirectory` | List directory contents | +| `search/usages` | Find references and implementations | +| `execute/runInTerminal` | Run a shell command | +| `execute/getTerminalOutput` | Get terminal output | +| `execute/testFailure` | Get test failure details | +| `web/fetch` | Fetch a web page | + +**Extension / MCP tools** can also be included using their identifier (e.g., `github/search_issues`). Use `/*` to include all tools from an MCP server. + +### Frontmatter syntax +```yaml +tools: ['read/readFile', 'search/codebase', 'edit/editFiles'] +``` + ## Example Output Structure (with skill reference) ```markdown @@ -63,6 +123,7 @@ Before generating the prompt, review the available skills in the `.github/skills name: my-new-prompt description: specialized task description argument-hint: input parameter hint +tools: ['edit/editFiles', 'read/readFile', 'search/codebase', 'execute/runInTerminal'] --- You are a specialized agent for... @@ -89,6 +150,7 @@ Use ${input:param1} to... name: my-new-prompt description: specialized task description argument-hint: input parameter hint +tools: ['read/readFile', 'search/codebase'] --- You are a specialized agent for... diff --git a/.github/prompts/generate-skill.prompt.md b/.github/prompts/generate-skill.prompt.md index 484fe2debc..e258adedc6 100644 --- a/.github/prompts/generate-skill.prompt.md +++ b/.github/prompts/generate-skill.prompt.md @@ -2,6 +2,8 @@ name: generate-skill description: Generate a GitHub Copilot Agent Skill (SKILL.md) following best practices and official documentation argument-hint: Describe the skill you want to create (e.g., "debugging SQL connection issues") +agent: agent +tools: ['read/readFile', 'edit/createFile', 'search'] --- You are an expert developer specialized in creating **GitHub Copilot Agent Skills**. diff --git a/.github/prompts/implement-feature.prompt.md b/.github/prompts/implement-feature.prompt.md index 4beae539fe..4623cb235c 100644 --- a/.github/prompts/implement-feature.prompt.md +++ b/.github/prompts/implement-feature.prompt.md @@ -62,9 +62,9 @@ Before writing code, produce a brief implementation plan covering: 4. Test against multiple SQL Server versions. ## 5. Write Tests -- **Unit tests** in `tests/UnitTests/` for isolated logic. -- **Functional tests** in `tests/FunctionalTests/` for API behavior without SQL Server. -- **Manual tests** in `tests/ManualTests/` for full integration with SQL Server. +- **Unit tests** in `src/Microsoft.Data.SqlClient/tests/UnitTests/` for isolated logic. +- **Functional tests** in `src/Microsoft.Data.SqlClient/tests/FunctionalTests/` for API behavior without SQL Server. +- **Manual tests** in `src/Microsoft.Data.SqlClient/tests/ManualTests/` for full integration with SQL Server. - Cover: - Happy path and edge cases - **Both sync and async code paths** where the feature exposes both variants diff --git a/.github/prompts/refine-test-overlap.prompt.md b/.github/prompts/refine-test-overlap.prompt.md index 824e18f145..f4d0339c27 100644 --- a/.github/prompts/refine-test-overlap.prompt.md +++ b/.github/prompts/refine-test-overlap.prompt.md @@ -1,7 +1,9 @@ --- -name: test-minimize-overlap +name: refine-test-overlap description: Run coverage overlap analysis and suggest test suite optimizations argument-hint: Test filter (e.g. FullyQualifiedName~MyTests) or describe the tests you want to analyze +agent: agent +tools: ['edit/editFiles', 'read/readFile', 'search/codebase', 'execute/runInTerminal', 'execute/getTerminalOutput'] --- You are an expert .NET Test Engineer specialized in optimizing test coverage and reducing technical debt. @@ -10,19 +12,19 @@ Your task is to analyze the user's test suite using the `AnalyzeTestOverlap.ps1` ## Skills This prompt leverages the following skills for specific sub-tasks: -- [generate-mstest-filter](../skills/generate-mstest-filter/SKILL.md) - For generating well-formed MSTest filter expressions +- [generate-mstest-filter](.github/skills/generate-mstest-filter/SKILL.md) - For generating well-formed MSTest filter expressions ## Tools -You have access to the analysis script at `[AnalyzeTestOverlap.ps1](./scripts/AnalyzeTestOverlap.ps1)`. +You have access to the analysis script at [AnalyzeTestOverlap.ps1](.github/prompts/scripts/AnalyzeTestOverlap.ps1). ## Workflow 1. **Parse or Generate Test Filter**: * If `${input:filter}` is a valid MSTest filter expression (e.g., `FullyQualifiedName~MyTests`), use it directly. - * If `${input:filter}` is a loose description (e.g., "connection tests" or "SqlCommand class"), follow the instructions in the [generate-mstest-filter](../skills/generate-mstest-filter/SKILL.md) skill to generate a proper filter expression. + * If `${input:filter}` is a loose description (e.g., "connection tests" or "SqlCommand class"), follow the instructions in the [generate-mstest-filter](.github/skills/generate-mstest-filter/SKILL.md) skill to generate a proper filter expression. * If `${input:filter}` is empty, ask the user for a test filter or description to target specific tests. 2. **Run Analysis**: - * Run the script using the filter: `.\scripts\AnalyzeTestOverlap.ps1 -Filter ""`. + * Run the script from the workspace root: `.\.github\prompts\scripts\AnalyzeTestOverlap.ps1 -Filter ""`. * *Note*: The script produces a console summary and a `test-coverage-analysis.json` file. 3. **Review Overlap**: diff --git a/.github/prompts/release-notes.prompt.md b/.github/prompts/release-notes.prompt.md index ca03933236..b800dfe245 100644 --- a/.github/prompts/release-notes.prompt.md +++ b/.github/prompts/release-notes.prompt.md @@ -1,19 +1,27 @@ --- name: release-notes description: Generate release notes for a specific milestone, covering all packages in the repository that have changes. -argument-hint: +argument-hint: agent: agent -tools: ['edit/createFile', 'edit/editFiles', 'read/readFile'] +tools: ['edit/createFile', 'edit/editFiles', 'read/readFile', 'execute/runInTerminal'] --- -Generate release notes for the milestone "${input:milestone}". +Generate release notes for the milestone "${input:milestone}" on the branch "${input:branch}". This repository ships multiple packages. Only generate release notes for packages that have relevant PRs in the milestone. All packages use the same template: [release-notes/template/release-notes-template.md](release-notes/template/release-notes-template.md). +## Branch Model + +The release notes content and the source code it describes live on different branches: + +- **Release notes files are maintained on `main`.** Every release's notes (for all branches/versions) are committed under `release-notes/` on `main`. Create and edit the release notes Markdown files on `main` (or a PR targeting `main`), not on the release branch. +- **The source code for the release lives only on the target branch `${input:branch}`.** Version sources (`Versions.props`), project files (`*.csproj`), and dependency files (`Directory.Packages.props`) reflect the released bits *as they exist on `${input:branch}`*, which can differ from `main`. When you look up versions, dependencies, TFM/OS scope, or verify API names (Steps 2.1, 2.2, 4, and the Version and Dependency Lookup table), read those source files from `${input:branch}` — not from your current `main` checkout. +- **Practical implication:** Do not assume a `...VersionDefault` or dependency version read from `main` matches what shipped on `${input:branch}`. Confirm against `${input:branch}` (e.g., `git show ${input:branch}:`), or against the milestone/release artifacts. + ## Package Registry | Package | Release Notes Directory | How to Identify PRs | -|---------|------------------------|---------------------| +| ------- | ----------------------- | ------------------- | | `Microsoft.Data.SqlClient` | `release-notes//` | Default — PRs not assigned to another package | | `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | `release-notes/add-ons/AzureKeyVaultProvider//` | Labels containing `AKV`, or PR titles/bodies/files referencing `AzureKeyVaultProvider`, `add-ons/`, or `AlwaysEncrypted.AzureKeyVaultProvider` | | `Microsoft.SqlServer.Server` | `release-notes/MSqlServerServer//` | PR titles/bodies/files referencing `Microsoft.SqlServer.Server` or `src/Microsoft.SqlServer.Server/` | @@ -21,20 +29,28 @@ This repository ships multiple packages. Only generate release notes for package | `Microsoft.Data.SqlClient.Extensions.Azure` | `release-notes/Extensions/Azure//` | PR titles/bodies/files referencing `Extensions.Azure` | | `Microsoft.Data.SqlClient.Internal.Logging` | `release-notes/Internal/Logging//` | PR titles/bodies/files referencing `Internal.Logging` | +> **Not all packages exist on every branch.** This table is the full, current package set. Older release branches ship a subset — for example, `release/6.1` and earlier have no extension packages (`Extensions.Abstractions`, `Extensions.Azure`) and no `Internal.Logging`; the companion-package set and even the `AzureKeyVaultProvider` source location vary by branch. Before generating notes for a package, confirm it actually exists on the target branch `${input:branch}` (e.g., `git ls-tree -r --name-only ${input:branch} | grep -i ""`). Skip any package that does not exist on `${input:branch}`, even if the table lists it. + ## Version and Dependency Lookup -Each package has its own versioning and dependency sources. Use these to determine package versions and dependency lists: +Each package's version and dependency information comes from MSBuild props/project files **on the target branch `${input:branch}`** (see Branch Model). The exact file paths, file names, and property names that hold versions **differ by branch**, because the versioning layout was refactored over time. Do not assume the layout of your current checkout — discover the version source on `${input:branch}`. + +Two known layouts: + +| Layout | Branches | MDS version source | Companion/extension version sources | +| ------ | -------- | ------------------ | ----------------------------------- | +| **Centralized** | `release/7.0` (and earlier 7.0.x) | `tools/props/Versions.props` (`MdsVersionDefault`) | `tools/props/Versions.props` imports per-package props with the older names: `…/Extensions/Abstractions/src/AbstractionsVersions.props`, `…/Extensions/Azure/src/AzureVersions.props`, `…/Internal/Logging/src/LoggingVersions.props`, `…/Microsoft.Data.SqlClient/add-ons/AzureKeyVaultProvider/AkvProviderVersions.props` | +| **Per-package** | `main`, `7.1+` | `src/Microsoft.Data.SqlClient/Versions.props` (`SqlClientVersionDefault`) | Each package has its own `Versions.props`: `…/Extensions/Abstractions/src/Versions.props` (`AbstractionsVersionDefault`), `…/Extensions/Azure/src/Versions.props` (`AzureVersionDefault`), `…/Internal/Logging/src/Versions.props` (`LoggingVersionDefault`), `…/AlwaysEncrypted.AzureKeyVaultProvider/src/Versions.props` (`AkvProviderVersionDefault`), `…/Microsoft.SqlServer.Server/Versions.props` (`SqlServerVersionDefault`) | + +Discovery approach (works regardless of layout): -| Package | Version Source | Dependency Source | -|---------|---------------|-------------------| -| `Microsoft.Data.SqlClient` | [tools/props/Versions.props](tools/props/Versions.props) (`MdsVersionDefault`) | [Directory.Packages.props](Directory.Packages.props) and the [project file](src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj) | -| `AzureKeyVaultProvider` | [tools/props/Versions.props](tools/props/Versions.props) (`AkvVersionDefault`) | [AKV project file](src/Microsoft.Data.SqlClient/add-ons/AzureKeyVaultProvider/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.csproj) and [Directory.Packages.props](Directory.Packages.props) | -| `Microsoft.SqlServer.Server` | [tools/props/Versions.props](tools/props/Versions.props) (`SqlServerPackageVersion`) | [SqlServer project file](src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj) | -| `Extensions.Abstractions` | [AbstractionsVersions.props](src/Microsoft.Data.SqlClient.Extensions/Abstractions/src/AbstractionsVersions.props) | [Abstractions.csproj](src/Microsoft.Data.SqlClient.Extensions/Abstractions/src/Abstractions.csproj) | -| `Extensions.Azure` | [AzureVersions.props](src/Microsoft.Data.SqlClient.Extensions/Azure/src/AzureVersions.props) | [Azure.csproj](src/Microsoft.Data.SqlClient.Extensions/Azure/src/Azure.csproj) | -| `Internal.Logging` | [LoggingVersions.props](src/Microsoft.Data.SqlClient.Internal/Logging/src/LoggingVersions.props) | [Logging.csproj](src/Microsoft.Data.SqlClient.Internal/Logging/src/Logging.csproj) | +1. List the version props on the target branch, e.g. `git ls-tree -r --name-only ${input:branch} | grep -i "Versions.props$"`. +2. Read the relevant file from the target branch, e.g. `git show ${input:branch}:`, and find the package's default/`PackageVersion` property. +3. Prefer the explicit shipped version: on a release branch the actual version may be supplied by the pipeline (`...PackageVersion`) rather than the `...VersionDefault` fallback, so confirm against the milestone/release artifacts rather than assuming the default. -Concrete dependency versions (e.g., `Azure.Core 1.49.0`) are centrally managed in [Directory.Packages.props](Directory.Packages.props). Framework-conditional versions (e.g., `net9.0` vs everything else) are handled by `Condition` attributes in the same file. +Dependency sources (read from `${input:branch}`): the per-package project file (`src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj`, the AKV `.csproj`, `Abstractions.csproj`, `Azure.csproj`, `Logging.csproj`, `Microsoft.SqlServer.Server.csproj`) plus the centrally-managed concrete versions in `Directory.Packages.props`. Framework-conditional versions (e.g., `net9.0` vs everything else) are handled by `Condition` attributes there. + +> **Companion package version alignment (7.0.2 and later):** Starting with 7.0.2, the companion packages (`AzureKeyVaultProvider`, `Extensions.Azure`, `Extensions.Abstractions`, `Internal.Logging`) ship version-aligned with the core `Microsoft.Data.SqlClient` driver. When generating notes for an aligned release, use the core MDS version (read from the target branch) for these companion packages — their per-package default version on `main` may point at a different next version and must not be assumed to be the shipped version. `Microsoft.SqlServer.Server` continues to version independently. ## Skills @@ -46,7 +62,8 @@ This prompt uses the following skill: ### 1. Fetch Milestone Items - Follow the instructions in the [fetch-milestone-prs](.github/skills/fetch-milestone-prs/SKILL.md) skill to fetch all merged PRs for the milestone "${input:milestone}". -- The output will be saved to `.milestone-prs/${input:milestone}/` with individual JSON files per PR and an `_index.json` summary. +- The output will be saved to `.milestone-prs/${input:milestone}/${input:branch}` with individual JSON files per PR and an `_index.json` summary. +- Identify any milestone items that don't have corresponding commits on the release branch "${input:branch}", and vice versa. ### 2. Analyze and Categorize @@ -57,6 +74,52 @@ This prompt uses the following skill: - Identify the contributors for the "Contributors" section. - **Assign each PR to one or more packages** using the identification rules in the Package Registry table. A PR may be relevant to multiple packages. PRs not matching any non-core package belong to `Microsoft.Data.SqlClient`. +### 2.1. Determine Target Framework (TFM) Scope Per Change + +For each PR included in release notes, determine whether it applies to all supported TFMs for the package or only a subset. + +Use source-level evidence (not assumptions) to classify scope: + +- **TFM-specific files** indicate scoped impact (for example, `.netfx.cs`, `.netcore.cs`). +- **Conditional compilation** indicates scoped impact (for example, `#if NETFRAMEWORK`, `#if NET`). +- **Project or build conditions** indicate scoped impact (for example, `Condition` expressions on `TargetFramework` or `TargetFrameworks`). +- **Tests-only TFM changes** should not be called out as customer-facing unless the behavior change is also present in product code. + +When writing notes: + +- If the change affects **all supported TFMs** for that package, do not add a TFM qualifier. +- If the change affects **only some TFMs**, include an explicit qualifier in the relevant bullet or section title. +- Use concise qualifiers like: + - `(net462 only)` + - `(net8.0/net9.0 only)` + +Do not infer TFM scope from labels alone; verify from changed files and code paths. + +### 2.2. Determine Operating System (OS) Scope Per Change + +For each PR included in release notes, determine whether it applies to all supported OS targets for the package or only a subset. + +Use source-level evidence (not assumptions) to classify scope: + +- **OS-specific files** indicate scoped impact (for example, `.windows.cs`, `.unix.cs`). +- **OS preprocessor symbols** indicate scoped impact (for example, `#if _WINDOWS`, `#if _UNIX`). +- **Project/build conditions** indicate scoped impact (for example, `TargetOs`, `NormalizedTargetOs`, or OS-conditional `ItemGroup`/`PropertyGroup` entries). +- **SNI implementation or native dependency gates** can imply OS scope when behavior changes only apply to native Windows SNI vs managed cross-platform paths. +- **Tests-only OS changes** should not be called out as customer-facing unless the behavior change is also present in product code. + +When writing notes: + +- If the change affects **all supported OS targets**, do not add an OS qualifier. +- If the change affects **only some OS targets**, include an explicit qualifier in the relevant bullet or section title. +- Use concise qualifiers like: + - `(Windows only)` + - `(Unix only)` + - `(Linux only)` + - `(macOS only)` +- If both TFM and OS are scoped, combine them in one qualifier, for example: `(net8.0/net9.0 on Windows only)`. + +Do not infer OS scope from labels alone; verify from changed files and code paths. + ### 3. Enrich Feature Sections with Issue Context For significant features or bug fixes that reference a GitHub issue: @@ -75,7 +138,7 @@ When release notes reference a public API (property, method, class): ### 5. Generate Release Notes for Each Package -For each package that has relevant PRs in the milestone: +For each package that ships in this milestone — i.e., it has relevant PRs, **or** it is a version-aligned companion package (7.0.2+) bumping to match the core `Microsoft.Data.SqlClient` release even without its own changes (see item 2): 1. **Determine the package version** using the Version Source from the lookup table above. Read the actual props/project file to find the version. @@ -83,9 +146,15 @@ For each package that has relevant PRs in the milestone: - Use the template from [release-notes/template/release-notes-template.md](release-notes/template/release-notes-template.md). - Fill in the template following the instructions in each section. - Only include sections (Added, Changed, Fixed, Removed) that have entries. + - For each Added/Changed/Fixed/Removed item, include TFM and OS scope qualifiers when Step 2.1 or Step 2.2 determines the change is not universal across the package's supported targets. - Look up dependencies using the Dependency Sources from the lookup table above. Resolve concrete versions from [Directory.Packages.props](Directory.Packages.props). - List dependencies per target framework. Use the project file's `` to determine which frameworks to list. - Omit the Contributors section for packages with no public contributors. + - **GA releases (all packages):** When the release is a stable (non-preview) version, structure the notes with two sections: + 1. **"Changes Since [last preview]"** — only the delta since the most recent preview of this package. + 2. **"Cumulative Changes Since [last stable]"** — all changes since the last stable release of this package, synthesized from all preview release notes plus the GA milestone. This applies to every package (MDS, AKV, Extensions.Azure, Abstractions, Internal.Logging, etc.), not just the core driver. Apply the cross-referencing from Step 3 to eliminate items already shipped in prior stable patch releases. + - **Preview releases:** Only include the delta since the previous release (preview or stable). No cumulative section is needed. + - **Version-alignment-only releases (7.0.2+ companion packages):** Starting with 7.0.2, the companion packages (`AzureKeyVaultProvider`, `Extensions.Azure`, `Extensions.Abstractions`, `Internal.Logging`) ship a new version aligned with the core driver on every core release — **even when they have no functional or API changes**. In that case, still create the package's release notes file using a version-alignment-only style: state that there are no functional or API changes, note the version alignment with the core driver, link to the core `Microsoft.Data.SqlClient ` notes, and (for .NET Framework) call out any `AssemblyVersion` strong-name change. Use the shipped 7.0.2 companion notes as the reference pattern (e.g., [release-notes/Extensions/Abstractions/7.0/7.0.2.md](release-notes/Extensions/Abstractions/7.0/7.0.2.md), [release-notes/Internal/Logging/7.0/7.0.2.md](release-notes/Internal/Logging/7.0/7.0.2.md), [release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.2.md](release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.2.md)). For the `Internal.Logging` package, retain its internal-use note. `Microsoft.SqlServer.Server` is **not** version-aligned and follows the normal skip rule. 3. **Create or update the version README** at `/README.md`. Follow the existing format — see [release-notes/add-ons/AzureKeyVaultProvider/6.1/README.md](release-notes/add-ons/AzureKeyVaultProvider/6.1/README.md) for reference: @@ -100,7 +169,9 @@ For each package that has relevant PRs in the milestone: | | | [Release Notes](.md) | ``` -4. **Skip packages without changes.** If a package has no relevant PRs in the milestone, do not create release notes for it. Report which packages had changes and which did not. +4. **Skip packages without changes (or that don't exist on the branch).** If a package has no relevant PRs in the milestone, or the package does not exist on the target branch `${input:branch}` (see the Package Registry note — older branches like `release/6.1` have no extension or `Internal.Logging` packages), do not create release notes for it. **Exception (7.0.2+):** version-aligned companion packages (`AzureKeyVaultProvider`, `Extensions.Azure`, `Extensions.Abstractions`, `Internal.Logging`) still get a release notes file when they bump to the aligned core version, even with no functional changes — use the version-alignment-only style from item 2. Report which packages had changes, which shipped alignment-only notes, which did not, and which are not present on the branch. + +5. **Cross-link companion packages from the core release notes.** When one or more companion packages (`AzureKeyVaultProvider`, `Extensions.Azure`, `Extensions.Abstractions`, `Internal.Logging`, `Microsoft.SqlServer.Server`) also ship in this milestone, add a `### Companion package release notes` section to the core `Microsoft.Data.SqlClient` release notes file that links to each companion package's release notes for the same version. This preserves context for the companion packages when the core release notes are used as the published GitHub release body. Use relative links (e.g., `../Extensions/Azure//.md`). Only list packages that actually shipped release notes in this milestone. ### 6. Update CHANGELOG.md @@ -115,8 +186,17 @@ For each package that has relevant PRs in the milestone: - If a section for the package doesn't yet exist, add one following the existing pattern (see the `AzureKeyVaultProvider` and `Microsoft.SqlServer.Server` sections for reference). - If the section already exists, add the new version link to its Release Information list. +### 8. Markdown for GitHub Release + +- Use the contents of the new release notes markdown file to produce markdown suitable for pasting into a GitHub UI Release textbox. + - GitHub renders newlines within paragraphs and lists as hard breaks, so remove those. + - Omit the main heading and first sub-heading. + - Update any relative links to use absolute URLs pointing to the file in the repository. + - Provide this new markdown in a code block that can easily be copied and pasted directly into the GitHub UI. + ## Notes -- Packages may ship as preview or stable independently. Use the actual version from the project/spec files. -- The directory structure mirrors existing conventions: `add-ons/AzureKeyVaultProvider/` for AKV, `MSqlServerServer/` for SqlServer, and `Extensions//` for the new extension packages. +- Release notes are maintained on `main` for all branches/releases, while the corresponding source code lives only on the target branch `${input:branch}` (see Branch Model above). Read version/dependency/source files from `${input:branch}`; write release notes files on `main`. +- Packages may ship as preview or stable independently. Use the actual version from the project/spec files on the target branch. +- The directory structure mirrors existing conventions: `add-ons/AzureKeyVaultProvider/` for AKV, `MSqlServerServer/` for SqlServer, `Extensions//` for the extension packages (e.g., `Extensions/Abstractions/`, `Extensions/Azure/`), and `Internal/Logging/` for the internal logging package. - When referencing code samples, link to files in the `doc/samples/` directory if a relevant sample exists. diff --git a/.github/prompts/review-pr-feedback.prompt.md b/.github/prompts/review-pr-feedback.prompt.md new file mode 100644 index 0000000000..c14a7b96cb --- /dev/null +++ b/.github/prompts/review-pr-feedback.prompt.md @@ -0,0 +1,123 @@ +--- +name: review-pr-feedback +description: Uses gh CLI to collect unresolved PR review feedback, optionally includes discussion comments, applies fixes, and reports status. +argument-hint: pr= repo= includeDiscussionComments= authorFilter= testScope= +tools: ['edit/editFiles', 'edit/createFile', 'read/readFile', 'read/problems', 'search/codebase', 'search/textSearch', 'search/fileSearch', 'execute/runInTerminal', 'execute/getTerminalOutput'] +--- +You are an expert software maintenance agent focused on resolving pull request feedback quickly, safely, and with clear traceability. + +## Context +- Workspace root: ${workspaceFolder} +- Target PR: ${input:pr} +- Optional repository override: ${input:repo} +- Include non-review discussion comments: ${input:includeDiscussionComments} +- Optional author filter: ${input:authorFilter} +- Optional focused testing hint: ${input:testScope} +- Optional selected context: ${selection} + +## Skills +#skill:generate-mstest-filter + +Use this skill when building a dotnet test filter: +- [generate-mstest-filter](.github/skills/generate-mstest-filter/SKILL.md) + +Follow the referenced skill instructions before producing any custom filter. + +## Task +1. Validate prerequisites +- Confirm gh CLI is installed and authenticated. +- Resolve repository from ${input:repo}, or infer from git remote. +- Resolve PR number from ${input:pr} (accept number or URL). +- Discover the correct git remote name from the current repository and store it for later commands. +- Use that discovered remote name for push and any other git operations that require a remote; do not assume `origin`. + +2. Gather actionable review feedback +- Query PR review threads with gh api GraphQL. +- Keep only unresolved threads where isResolved is false. +- Extract thread id, file path, line/startLine, comment url, author login, and body. +- If ${input:authorFilter} is provided, apply it case-insensitively. + +3. Optionally gather non-review discussion comments +- If ${input:includeDiscussionComments} is true, fetch PR issue comments. +- Mark these as Informational because they do not have open/resolved state. +- Apply ${input:authorFilter} if provided. + +4. Build an implementation plan +- Group unresolved review feedback by file and risk. +- Determine minimal safe edits needed. +- Identify comments that are non-actionable or ambiguous. +- Ask the user to confirm the plan before proceeding, showing a concise summary of proposed changes and rationale. + +5. Implement and verify +- Apply required code or test updates with smallest safe change set. +- Run targeted checks first. +- If ${input:testScope} is provided, generate and use a focused MSTest filter via the skill. +- Collect diagnostics when tests cannot run. + +6. Classify each item +- Fixed: change implemented and validated. +- Needs Clarification: ambiguous, conflicting, or insufficiently specified. +- Blocked: external dependency, permission, or missing context. +- Informational: non-review discussion comment captured only. + +7. Produce a final report +- Keep review-thread outcomes and discussion outcomes in separate sections. +- Include evidence for each item: file location, change summary, validation result. +- Draft a distinct reply for each comment item that addresses that exact comment's request, context, and outcome. + +8. Commit changes +- If any changes were made, create a commit with a clear message referencing the PR and summarizing the resolution. +- Prompt the user to review and confirm the commit message before finalizing. +- When suggesting or performing a push, use the discovered git remote name. +- Prompt the user to push the commit if they have permissions, or provide instructions if they do not. +- Prompt the user to reply to each original PR comment with a comment-specific response and link to the relevant commit or code location, if appropriate. +- Prompt the user to mark review threads as resolved in GitHub if they have permissions, or provide instructions if they do not. + +## Output Format +1. PR Scope +- Repo +- PR number +- Unresolved review threads found +- Discussion comments found (if enabled) + +2. Unresolved Review Feedback (Actionable) +- Item: +- Location: : +- Author: +- Request summary: +- Action taken: +- Status: Fixed | Needs Clarification | Blocked +- Evidence: +- Suggested reply: + +3. Discussion Comments (Informational, optional) +- Item: +- Author: +- Summary: +- Notes: +- Suggested reply: + +4. Validation +- Commands run +- Filters used +- Pass/fail summary +- Remaining warnings/errors + +5. Final Summary +- Files changed +- Number fixed +- Number needing clarification +- Number blocked +- Number informational +- Recommended next step + +## Rules +- Do not invent comments; only act on data fetched from gh. +- Review-thread resolution tracking is authoritative for unresolved state. +- Keep behavior-compatible edits unless feedback explicitly requires change. +- If no unresolved review threads exist, report that explicitly. +- If auth or permission fails, report exact failure and minimum required user action. +- Do not use `set -e` in bash commands or scripts. +- After each terminal step, verify the bash session is still alive; if it died, report it immediately, start a new session, and continue from the last confirmed checkpoint. +- Use the discovered git remote name consistently anywhere a remote is required. +- Do not post generic batch replies; each reply must be tailored to the specific comment content and its exact resolution status. diff --git a/.github/prompts/triage-pipeline-failures.prompt.md b/.github/prompts/triage-pipeline-failures.prompt.md new file mode 100644 index 0000000000..98c223c3d5 --- /dev/null +++ b/.github/prompts/triage-pipeline-failures.prompt.md @@ -0,0 +1,185 @@ +--- +name: triage-pipeline-failures +description: Find and classify failing tests in the CI/CD pipelines at or after a given commit, then fix or quarantine them. +argument-hint: [optional scope, e.g. specific pipelines/branches] +agent: agent +# No `tools:` scoping on purpose: this prompt is access-agnostic and must be able +# to call whatever Azure DevOps MCP server is connected (e.g. `ado/*`) in addition +# to the built-in terminal/read/search/edit tools. Declaring a scoped `tools:` list +# would strip out MCP/extension tools and break the preferred ADO MCP access path. +--- + +Triage failing tests in the CI/CD pipelines for commit +`${input:commit}` and later. Treat only the **first whitespace-delimited token** of +`${input:commit}` as the target commit SHA — that token is what every git ancestry +check (`git merge-base --is-ancestor ...`) uses. Any remaining text is +**optional scope** (e.g. a pipeline name or branch): honor it when present, otherwise +use the defaults below. + +## Azure DevOps access is agnostic + +Every data-retrieval step below is described as an **operation**, not a command. +Perform each operation with whatever Azure DevOps access is available, in this order +of preference: + +1. An **Azure DevOps MCP server**, if one is connected (preferred — no shell needed). +2. The **`az` CLI** (`az rest --resource ...`, + `az pipelines ...`, `az boards ...`). +3. **Direct ADO REST** calls over HTTPS with a bearer token. + +Do not assume a specific mechanism. If the first choice is unavailable or errors, +fall back to the next. Keep read operations read-only; only edits to test source +files (quarantine/fix) modify state, and those happen in the repo, not in ADO. + +## Environment + +- **ADO organization**: ``. +- **Projects**: + - `` — CI/PR pipelines target the upstream GitHub repo. + - `` — CI/OneBranch pipelines target the ADO mirror repo. +- The ADO mirror repo preserves the **same commit SHAs** as GitHub, so git + ancestry against a GitHub SHA works. It **lags** GitHub because synchronization is + gated by PRs: a commit only appears in the mirror once its sync PR completes, so a + target commit may not be present in the mirror yet even though it is on GitHub. + +## Step 1 — Scope to the right pipelines + +**Operation:** list build definitions in both `public` and `ADO.Net`, with each +definition's folder path, `queueStatus`, and `repository.type` / `repository.name`. + +Ignore any definition that is **not currently enabled** (`queueStatus != enabled`, +i.e. disabled or paused) — also skip names flagged `[Disabled]`, `[Retired]`, or under +`\Retired\` folders. Then keep only definitions whose repo is the upstream GitHub repo +or the ADO mirror repo. Exclude legacy driver repos and native SNI repos unless the +user asks for SNI. + +Because this triage targets **non-PR commit runs** (see Step 2), prefer CI/branch +definitions over PR-validation ones. Prefer CI, package, stress, Kerberos, +Managed-Instance, and OneBranch official/non-official definitions across both projects. +PR-triggered definitions are in scope only for the CI/branch runs they may also host — +their PR-ref runs are excluded in Step 2 unless the user asks to include PR runs. + +## Step 2 — Find runs at/after the target commit + +**Operation:** for each in-scope definition, list recent runs (filter to +`failed`, `partiallySucceeded`, `canceled`) with their `sourceBranch`, +`sourceVersion`, `result`, and `finishTime`. + +**Limit to non-PR commit runs.** Only consider runs triggered by real commits on +tracked branches (e.g. `refs/heads/main`, `refs/heads/release/*`); **exclude PR +validation runs**. A run is a PR run — and therefore out of scope — when any of these +hold: + +- Its `sourceBranch` is an ephemeral merge ref such as `refs/pull/N/merge` or + `refs/pull/N/head`. +- Its build `reason` is `pullRequest`. +- It is a PR-triggered definition running against a PR ref. + +Keep only runs whose `sourceVersion` is a committed SHA on a tracked branch. If the +user explicitly asks to include PR runs, honor that override. + +Resolve **"at or after `${input:commit}`" by commit graph, not timestamp**: + +- `git merge-base --is-ancestor ` → true means the run's + commit is the target or a descendant (in scope). +- `sourceVersion == ` → the target itself (in scope). +- Divergent `release/*` or `dev/*` commits do **not** descend from a `main` target — + exclude them. + +Mirror runs use the **same SHAs** as GitHub, so apply the same +ancestry checks. Because mirror sync is PR-gated, the target commit may not have +reached the mirror yet — in that window there simply are no in-scope mirror runs, so +do not infer a run is out of scope from a SHA mismatch (there is none); it is only a +timing lag. + +## Step 3 — Enumerate failing test runs per build + +**Operation:** for each in-scope build, list its test runs. + +Compute real failures as `totalTests - passedTests - notApplicableTests`. Do **not** +treat `unanalyzedTests`/`notApplicableTests` as failures. Keep runs with failures > 0. + +## Step 4 — Get failing test names and errors + +**Operation:** for each failing test run, fetch the `Failed` results with their +`automatedTestName`, `errorMessage`, and `stackTrace`. + +**You must capture the actual xUnit output and full stack trace for every failed +test — do not classify a failure without it.** The one-line `errorMessage` is not +enough; get the complete assertion text (e.g. `Assert.Equal() Failure: Values differ / +Expected / Actual`) and the full stack frames (the test method and the failing product +frames). If any source truncates it, cross-check another until you have the whole thing: + +- The result's `errorMessage` + `stackTrace` fields (expand sub-results — see below). +- The test run's **attachments** (TRX / `*.trx`, console logs) when the API truncates + long stacks. +- The **job log** for the test step (Step 5) — the raw `dotnet test` / xUnit output + always contains the assertion and stack, even when the results API does not. + +**CRITICAL — data-driven (Theory) results hide the error on a child:** xUnit +`[Theory]`/`[ClassData]`/`[InlineData]` tests publish as a parent result with +`resultGroupType == "dataDriven"` whose own `errorMessage`/`stackTrace` are **null**. +The real assertion lives on the failing **sub-result**. When a `Failed` result has a +null error, re-fetch that result **including sub-results** and read the child. A null +parent error means "look at the children", not "the test aborted". Only treat it as an +abort when the build log also shows no assertion and the process was terminated +(e.g. a `--blame-hang` dump). + +## Step 5 — Locate each failure's job (for logs/links) + +**Operation:** for a failing run, read its `pipelineReference` (stage/phase/job), then +read the build's timeline and walk Stage → Phase → Job by `parentId` to get the job +record id. Build deep links: + +- Tests tab: `.../_build/results?buildId=&view=ms.vss-test-web.build-test-results-tab` +- A specific result: append `&runId=&resultId=&paneView=debug` +- Job logs: `.../_build/results?buildId=&view=logs&j=` + +## Step 6 — Classify every failure + +| Class | Signals | Action | +|-------|---------|--------| +| **True positive** (broken driver) | Deterministic; fails on every leg for the commit; assertion tied to changed code; absent on the parent commit | Fix the bug; keep/add a failing test | +| **Test-isolation / concurrency** | Off-by-a-small-count on a process-global resource (e.g. pool `ConnectionCount` Expected 2 Actual 3); some data rows pass, others fail; depends on parallel tests | Isolate the resource (unique connection string, `[Collection]`); else quarantine | +| **Flaky (timing/GC/load)** | Intermittent; only under CI load; GC-finalizer or retry/failover timing; "connection is broken" under contention | Deterministic fix (poll not sleep, set retry interval/timeouts); else quarantine | +| **Environmental / infra** | Empty error AND no assertion in the log; host/agent crash; blame-hang dump; network/DTC outage; many unrelated tests fail at once | Re-run to confirm; report infra; don't quarantine on a single infra hit | + +Determine **regression vs pre-existing** by repeating Steps 3–4 on the +immediately-preceding in-scope build (the parent commit). A failure present before +`${input:commit}` was not introduced by it. + +## Step 7 — Check quarantine status before acting + +A test is already quarantined if it carries `[Trait("category", "flaky")]`; those run +in a separate, non-blocking quarantine step (`TestFilters="category=flaky"`) while the +regular step excludes `category!=failing&category!=flaky&category!=interactive`. + +- Already-quarantined failure = expected quarantine noise, not a blocker. Only escalate + with a real fix. +- Non-quarantined failure in a regular step = a real blocker. + +## Step 8 — Present findings and STOP (checkpoint) + +Steps 1–7 are **read-only investigation**. Before changing anything, present your +findings and wait for the user's explicit go-ahead. Do **not** edit any files or take +any action until the user approves. + +Present a per-failure table: test name, in-scope build(s), full xUnit assertion + +key stack frames, classification, whether it is a regression, current quarantine +status, and the **proposed** action (fix / quarantine / already quarantined / +re-run to confirm). Link each failure to its build/result and job. Then explicitly ask +the user which items to act on. + +## Step 9 — Act (only after approval) + +For each item the user approves: + +1. Prefer a **deterministic fix** that removes the race/isolation/timing dependency. +2. Otherwise **quarantine**: add `[Trait("category", "flaky")]` plus a comment holding + the observed failure signature (test name, assertion, key stack frames) and the + root-cause reasoning. Mirror the style of existing quarantine comments in the test suite. +3. Cover both sync and async variants when the API has both. +4. Un-quarantine once fixed and consistently green. + +Make only the source edits needed to fix or quarantine; do not modify pipeline YAML or +ADO state. After editing, report what changed. diff --git a/.github/prompts/update-build-pipelines.prompt.md b/.github/prompts/update-build-pipelines.prompt.md index 617cf39901..a652c9f82f 100644 --- a/.github/prompts/update-build-pipelines.prompt.md +++ b/.github/prompts/update-build-pipelines.prompt.md @@ -3,7 +3,7 @@ name: update-build-pipelines description: Guided workflow for updating Azure DevOps CI/CD pipelines for Microsoft.Data.SqlClient. argument-hint: agent: agent -tools: ['edit/createFile', 'edit/editFiles', 'read/readFile', 'codebase/search'] +tools: ['edit/createFile', 'edit/editFiles', 'read/readFile', 'search'] --- Update the Azure DevOps build pipelines for: "${input:change}". @@ -15,35 +15,40 @@ Follow this workflow step-by-step: ## 1. Understand the Pipeline Architecture - Read the relevant pipeline file(s) in `eng/pipelines/`. - Key pipelines: - - `dotnet-sqlclient-ci-core.yml` — Core CI pipeline (reusable by reference pipelines) + - `dotnet-sqlclient-ci-core.yml` — Core CI pipeline template used by CI and PR definitions - `dotnet-sqlclient-ci-project-reference-pipeline.yml` — CI with project references - `dotnet-sqlclient-ci-package-reference-pipeline.yml` — CI with package references - `sqlclient-pr-project-ref-pipeline.yml` — PR validation (project references) - `sqlclient-pr-package-ref-pipeline.yml` — PR validation (package references) - - `dotnet-sqlclient-signing-pipeline.yml` — Package signing - - `akv-official-pipeline.yml` — AKV provider official build (1ES/OneBranch) - - `stress-tests-pipeline.yml` — Stress tests + - `onebranch/sqlclient-official.yml` — official OneBranch build/release pipeline + - `onebranch/sqlclient-non-official.yml` — non-official OneBranch build/release pipeline + - `ci/stress/sqlclient-ci-stress-pipeline.yml` — stress test pipeline - Shared templates live in `eng/pipelines/common/templates/` (jobs/, stages/, steps/). -- Variables are defined in `eng/pipelines/variables/` and `eng/pipelines/libraries/`. +- CI variables are defined in `eng/pipelines/libraries/`; OneBranch variables are defined in `eng/pipelines/onebranch/variables/`. ## 2. Identify What Needs to Change - Determine which pipeline files are affected. - Check if the change impacts shared templates that are reused across multiple pipelines. - Identify if new parameters, variables, or stages need to be added. - Review existing parameters to understand the current configuration surface: - - `targetFrameworks` / `targetFrameworksUnix` — test target frameworks + - `targetFrameworks` / `targetFrameworksUnix` — Windows and Unix test TFMs + - `netcoreVersionTestUtils` — runtime used by shared test utilities - `referenceType` — Project or Package reference - `buildConfiguration` — Debug/Release - `useManagedSNI` — Managed vs Native SNI testing + - `runLegacySqlTests` — whether to include SQL Server 2016/2017 legs ## 3. Implement the Change - Modify YAML files following the existing patterns and indentation style. - When adding new stages, follow the existing stage ordering: - 1. `build_abstractions_package_stage` - 2. `build_sqlclient_package_stage` - 3. `build_azure_package_stage` - 4. `stress_tests_stage` (optional) - 5. `run_tests_stage` + 1. `generate_secrets` + 2. `build_sqlserver_package_stage` + 3. `build_logging_package_stage` + 4. `build_abstractions_package_stage` + 5. `build_sqlclient_package_stage` + 6. `build_azure_package_stage` + 7. `verify_nuget_packages_stage` + 8. `ci_run_tests_stage` - When adding new test parameters, ensure they are wired through to test execution steps. - When modifying shared templates, verify all consuming pipelines still work. @@ -52,6 +57,7 @@ Follow this workflow step-by-step: - Test filters by platform: `nonnetfxtests`, `nonnetcoreapptests`, `nonwindowstests`, `nonlinuxtests`. - SNI testing matrix: both Native (`useManagedSNI=false`) and Managed (`useManagedSNI=true`). - Always Encrypted tests controlled by `runAlwaysEncryptedTests` parameter. +- Stress coverage is maintained under `eng/pipelines/ci/stress/`, not as a stage inside `dotnet-sqlclient-ci-core.yml`. ## 5. Validate - Verify YAML syntax is valid. diff --git a/.github/scripts/auto-assign-pr.js b/.github/scripts/auto-assign-pr.js new file mode 100644 index 0000000000..0f2e77d3c1 --- /dev/null +++ b/.github/scripts/auto-assign-pr.js @@ -0,0 +1,133 @@ +// Auto-assign PR load balancer. +// +// Selects up to 2 assignees for a qualifying PR from a configurable pool, +// balancing by current open-PR assignment count. Invoked by the +// `auto-assign-pr.yml` workflow via `actions/github-script`. +module.exports = async ({ github, context, core }) => { + const owner = context.repo.owner; + const repo = context.repo.repo; + const prNumber = context.issue.number; + const author = context.payload.pull_request.user.login; + const normalizeLogin = login => login.toLowerCase(); + const parseCsvLogins = value => (value ?? '') + .split(',') + .map(entry => entry.trim()) + .filter(entry => entry.length > 0); + + // Fallback pool keeps behavior unchanged when no repo variable is configured. + const defaultPool = ['cheenamalhotra', 'paulmedynski', 'priyankatiwari08', 'benrr101', 'mdaigle', 'apoorvdeshmukh']; + const configuredPool = parseCsvLogins(process.env.PR_REVIEWER_POOL); + const rawPool = configuredPool.length > 0 ? configuredPool : defaultPool; + const seenPoolUsers = new Set(); + const pool = []; + for (const user of rawPool) { + const normalized = normalizeLogin(user); + if (!seenPoolUsers.has(normalized)) { + seenPoolUsers.add(normalized); + pool.push(user); + } + } + + let latestPr; + try { + const response = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber + }); + latestPr = response.data; + } catch (error) { + throw new Error(`Failed to fetch latest PR details: ${error.message}`); + } + + if (latestPr.state !== 'open' || latestPr.draft || !latestPr.milestone) { + console.log('PR is no longer assignment-eligible (not open, draft, or missing milestone).'); + return; + } + + const currentAssignees = (latestPr.assignees ?? []).map(a => a.login); + + console.log(`PR Author: ${author}`); + console.log(`Event Name: ${context.eventName}; Is Fork PR: ${context.payload.pull_request.head.repo.fork === true}`); + console.log(`Current Assignees: ${currentAssignees.join(', ')}`); + + if (currentAssignees.length >= 2) { + console.log('PR already has 2 or more assignees. No action needed.'); + return; + } + + const neededAssigneesCount = 2 - currentAssignees.length; + + const candidates = pool.filter(user => + normalizeLogin(user) !== normalizeLogin(author) && + !currentAssignees.some(a => normalizeLogin(a) === normalizeLogin(user)) + ); + + if (candidates.length === 0) { + console.log('No valid candidates left in the pool.'); + return; + } + + const workloads = {}; + const canonicalCandidateByNormalized = {}; + candidates.forEach(user => { + const normalized = normalizeLogin(user); + workloads[normalized] = 0; + canonicalCandidateByNormalized[normalized] = user; + }); + + try { + // Rank candidates by current assignment count across all open PRs. + const iterator = github.paginate.iterator(github.rest.pulls.list, { + owner, + repo, + state: 'open', + per_page: 100 + }); + + for await (const response of iterator) { + for (const pr of response.data) { + if (pr.draft) continue; + if (pr.assignees) { + for (const assignee of pr.assignees) { + const login = normalizeLogin(assignee.login); + if (workloads[login] !== undefined) { + workloads[login]++; + } + } + } + } + } + } catch (error) { + throw new Error(`Failed to fetch open PRs for auto-assignment: ${error.message}`); + } + + const workloadArray = candidates.map(user => { + const normalized = normalizeLogin(user); + return { user: canonicalCandidateByNormalized[normalized], count: workloads[normalized] }; + }); + console.log('Current Workloads:', workloadArray); + + // Shuffle before sorting so ties are broken fairly instead of favoring pool order. + for (let i = workloadArray.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [workloadArray[i], workloadArray[j]] = [workloadArray[j], workloadArray[i]]; + } + + workloadArray.sort((a, b) => a.count - b.count); + + const selectedAssignees = workloadArray.slice(0, neededAssigneesCount).map(w => w.user); + console.log(`Selected candidates: ${selectedAssignees.join(', ')}`); + + if (selectedAssignees.length === 0) { + console.log('No assignees selected. No action needed.'); + return; + } + + await github.rest.issues.addAssignees({ + owner, + repo, + issue_number: prNumber, + assignees: selectedAssignees + }); +}; diff --git a/.github/scripts/check-milestone-branch.sh b/.github/scripts/check-milestone-branch.sh new file mode 100755 index 0000000000..3be77eec59 --- /dev/null +++ b/.github/scripts/check-milestone-branch.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# check-milestone-branch.sh +# +# Validates that a pull request's milestone is consistent with the branch the +# pull request targets. +# +# OVERVIEW +# -------- +# Milestones in this repository are named "..", optionally +# with a pre-release suffix (e.g. "7.0.3", "8.0.0-preview1"). Every milestone +# therefore maps to a candidate release branch: +# +# .. -> release/. +# +# Release branches and configured milestones determine where the work belongs: +# +# * The branch EXISTS -> that version has already forked off the default +# branch and is in servicing. Changes for it go to release/.. +# +# * The branch DOES NOT exist, and this is the earliest configured milestone +# series without a release branch -> that version is in development on the +# default branch. Changes for it go to the default branch. +# +# * Any other series -> the default branch carries exactly one development +# line, so a later series is not active yet and an earlier series is no +# longer in development. Neither may target the default branch. +# +# This rule is self-maintaining: no hard-coded version list needs updating when +# a new release branch is cut. +# +# VALIDATION MATRIX +# ----------------- +# Target branch Release branch exists? Result +# ----------------------- ----------------------- ---------------------- +# release/. n/a (it is the target) pass +# another release/* n/a fail (mismatch) +# default branch no, active line pass +# default branch no, later configured line fail (not active yet) +# default branch no, earlier configured line fail (no longer in development) +# default branch no active line configured fail (milestone missing) +# default branch yes fail (needs servicing branch) +# anything else n/a skipped (integration branch) +# +# Pull requests into long-lived integration branches (e.g. "dev/paul/foo") are +# skipped, because the milestone is enforced when that branch is merged into +# the default branch or a release branch. +# +# Milestones that don't parse as ".." are skipped with a +# notice rather than failing the build. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# MILESTONE_TITLE The PR's milestone title (e.g. "7.0.3"). +# BASE_REF The branch the PR targets (e.g. "main", "release/7.0"). +# DEFAULT_BRANCH The repository's default branch (e.g. "main"). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# GH_TOKEN GitHub token for API calls (gh CLI auth). +# +# OUTPUTS +# ------- +# Emits ::notice:: on success/skip and ::error:: on failure. +# Exits 0 when the milestone and target branch agree (or the check is +# skipped), and 1 when they conflict. +# +# USAGE +# Called from the check-milestone.yml workflow. Can also be run locally: +# +# export MILESTONE_TITLE="7.0.3" +# export BASE_REF="main" +# export DEFAULT_BRANCH="main" +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/check-milestone-branch.sh +# +################################################################################# +set -euo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${MILESTONE_TITLE:?MILESTONE_TITLE environment variable is required}" +: "${BASE_REF:?BASE_REF environment variable is required}" +: "${DEFAULT_BRANCH:?DEFAULT_BRANCH environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +# -- Derive the candidate release branch from the milestone ------------------- +# Accepts "X.Y.Z" with an optional pre-release/build suffix, e.g. "8.0.0-preview1". +if [[ "${MILESTONE_TITLE}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-+].*)?$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" +else + echo "::notice::Milestone '${MILESTONE_TITLE}' is not in 'major.minor.patch' form; skipping the target branch check." + exit 0 +fi + +RELEASE_BRANCH="release/${MAJOR}.${MINOR}" + +# -- Skip integration branches ------------------------------------------------ +# Only the default branch and release branches carry milestone semantics. +if [[ "${BASE_REF}" != "${DEFAULT_BRANCH}" && "${BASE_REF}" != release/* ]]; then + echo "::notice::PR targets integration branch '${BASE_REF}'; skipping the milestone/branch check." + exit 0 +fi + +# -- Validate a release branch target ----------------------------------------- +# The target branch itself proves which version is being serviced, so no branch +# listing is needed here. +if [[ "${BASE_REF}" == release/* ]]; then + if [[ "${BASE_REF}" != "${RELEASE_BRANCH}" ]]; then + echo "::error::Milestone '${MILESTONE_TITLE}' belongs to '${RELEASE_BRANCH}', but this PR targets '${BASE_REF}'. Retarget the PR or assign the milestone that matches '${BASE_REF}'." + exit 1 + fi + + echo "::notice::Milestone '${MILESTONE_TITLE}' matches target branch '${BASE_REF}'." + exit 0 +fi + +# -- Validate a default branch target ----------------------------------------- +# 'matching-refs' returns only refs under the given prefix, so this is a single +# cheap call regardless of how many topic branches the repository has. +if ! RELEASE_REFS=$(gh api "repos/${GITHUB_REPOSITORY}/git/matching-refs/heads/release/" \ + --jq '.[].ref' 2>&1); then + echo "::error::Unable to list release branches for '${GITHUB_REPOSITORY}': ${RELEASE_REFS}" + exit 1 +fi + +if grep -qxF "refs/heads/${RELEASE_BRANCH}" <<< "${RELEASE_REFS}"; then + echo "::error::Milestone '${MILESTONE_TITLE}' is a servicing release owned by '${RELEASE_BRANCH}', but this PR targets '${DEFAULT_BRANCH}'. Either retarget the PR to '${RELEASE_BRANCH}', or assign an in-development milestone and add the 'Hotfix ${MAJOR}.${MINOR}.${PATCH}' label so the change is cherry-picked after merge." + exit 1 +fi + +if ! MILESTONES=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/milestones?state=all&per_page=100" \ + --jq '.[].title' 2>&1); then + echo "::error::Unable to list milestones for '${GITHUB_REPOSITORY}': ${MILESTONES}" + exit 1 +fi + +ACTIVE_MAJOR="" +ACTIVE_MINOR="" +LATEST_RELEASE_MAJOR="" +LATEST_RELEASE_MINOR="" +while IFS= read -r release_ref; do + if [[ ! "${release_ref}" =~ ^refs/heads/release/([0-9]+)\.([0-9]+)$ ]]; then + continue + fi + + release_major="${BASH_REMATCH[1]}" + release_minor="${BASH_REMATCH[2]}" + if [[ -z "${LATEST_RELEASE_MAJOR}" ]] || + (( 10#${release_major} > 10#${LATEST_RELEASE_MAJOR} )) || + (( 10#${release_major} == 10#${LATEST_RELEASE_MAJOR} && 10#${release_minor} > 10#${LATEST_RELEASE_MINOR} )); then + LATEST_RELEASE_MAJOR="${release_major}" + LATEST_RELEASE_MINOR="${release_minor}" + fi +done <<< "${RELEASE_REFS}" + +while IFS= read -r milestone; do + if [[ ! "${milestone}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-+].*)?$ ]]; then + continue + fi + + candidate_major="${BASH_REMATCH[1]}" + candidate_minor="${BASH_REMATCH[2]}" + candidate_branch="release/${candidate_major}.${candidate_minor}" + if grep -qxF "refs/heads/${candidate_branch}" <<< "${RELEASE_REFS}"; then + continue + fi + + if [[ -n "${LATEST_RELEASE_MAJOR}" ]] && + { (( 10#${candidate_major} < 10#${LATEST_RELEASE_MAJOR} )) || + (( 10#${candidate_major} == 10#${LATEST_RELEASE_MAJOR} && 10#${candidate_minor} <= 10#${LATEST_RELEASE_MINOR} )); }; then + continue + fi + + if [[ -z "${ACTIVE_MAJOR}" ]] || + (( 10#${candidate_major} < 10#${ACTIVE_MAJOR} )) || + (( 10#${candidate_major} == 10#${ACTIVE_MAJOR} && 10#${candidate_minor} < 10#${ACTIVE_MINOR} )); then + ACTIVE_MAJOR="${candidate_major}" + ACTIVE_MINOR="${candidate_minor}" + fi +done <<< "${MILESTONES}" + +if [[ -z "${ACTIVE_MAJOR}" ]]; then + echo "::error::No configured milestone series is newer than the newest release branch, so no development line is active on '${DEFAULT_BRANCH}'. Create the milestone for the next version before targeting '${DEFAULT_BRANCH}' with '${MILESTONE_TITLE}'." + exit 1 +fi + +if (( 10#${MAJOR} != 10#${ACTIVE_MAJOR} || 10#${MINOR} != 10#${ACTIVE_MINOR} )); then + if (( 10#${MAJOR} > 10#${ACTIVE_MAJOR} || + (10#${MAJOR} == 10#${ACTIVE_MAJOR} && 10#${MINOR} > 10#${ACTIVE_MINOR}) )); then + echo "::error::Milestone '${MILESTONE_TITLE}' is for a later development line, but the ${ACTIVE_MAJOR}.${ACTIVE_MINOR} milestone series remains active on '${DEFAULT_BRANCH}' until 'release/${ACTIVE_MAJOR}.${ACTIVE_MINOR}' is cut. Assign a milestone from the active ${ACTIVE_MAJOR}.${ACTIVE_MINOR} line." + else + echo "::error::Milestone '${MILESTONE_TITLE}' is for the ${MAJOR}.${MINOR} line, which is no longer in development on '${DEFAULT_BRANCH}'; the active line is ${ACTIVE_MAJOR}.${ACTIVE_MINOR}. Assign a milestone from the active ${ACTIVE_MAJOR}.${ACTIVE_MINOR} line." + fi + exit 1 +fi + +echo "::notice::Milestone '${MILESTONE_TITLE}' is still in development (no '${RELEASE_BRANCH}' branch); targeting '${DEFAULT_BRANCH}' is correct." diff --git a/.github/scripts/cherry-pick-to-release.sh b/.github/scripts/cherry-pick-to-release.sh new file mode 100755 index 0000000000..69a655d110 --- /dev/null +++ b/.github/scripts/cherry-pick-to-release.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# cherry-pick-to-release.sh +# +# Cherry-picks a merge commit from the default branch onto a release branch +# and opens a pull request for the result. If the cherry-pick conflicts, an +# empty-commit placeholder PR is created with manual resolution instructions. +# +# OVERVIEW +# -------- +# This script performs the following steps: +# +# 1. Derive the target release branch from the version's major.minor +# (e.g. "7.0.1" → release/7.0). +# +# 2. Check whether the commit's patch is already present on the target +# branch (via 'git cherry'). If so, exit cleanly — nothing to do. +# +# 3. Detect whether the merge commit is a true merge (2+ parents) or a +# squash-merge (1 parent). True merges require '--mainline 1'. +# +# 4. Attempt the cherry-pick: +# - On success: push the branch, look up the milestone, create a PR. +# - On conflict: abort, push an empty-commit placeholder, create a +# "CONFLICTS" PR with manual resolution instructions. +# +# 5. Milestone lookup is best-effort. If the milestone doesn't exist yet +# the PR is created without one and a warning note is added to the body. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# VERSION Full hotfix version, e.g. "7.0.1". +# MERGE_COMMIT_SHA SHA of the merge commit on the default branch. +# PR_NUMBER Number of the original PR that was merged. +# PR_TITLE Title of the original PR (used in cherry-pick PR title). +# GH_TOKEN GitHub token for 'gh' CLI authentication. +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set by Actions. +# +# OUTPUTS +# ------- +# On success or conflict, a new PR is created on GitHub. +# On already-applied, the script exits 0 with a notice. +# +# USAGE +# Typically called from the cherry-pick-hotfix.yml workflow. +# The git working directory must have full history (fetch-depth: 0) and +# user.name / user.email must be configured before calling this script. +# +# Local testing example (dry-run — comment out 'gh pr create' calls): +# +# export VERSION="7.0.1" +# export MERGE_COMMIT_SHA="abc123" +# export PR_NUMBER=42 +# export PR_TITLE="Fix connection timeout" +# export GH_TOKEN="ghp_..." +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/cherry-pick-to-release.sh +# +################################################################################# +set -euo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${VERSION:?VERSION environment variable is required}" +: "${MERGE_COMMIT_SHA:?MERGE_COMMIT_SHA environment variable is required}" +: "${PR_NUMBER:?PR_NUMBER environment variable is required}" +: "${PR_TITLE:?PR_TITLE environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +# -- Step 1: Derive target branch from major.minor --------------------------- +# "7.0.1" → "7.0", so target branch is "release/7.0". +# Use a bash regex for portability (grep -P is not available on macOS BSD grep). +if [[ "${VERSION}" =~ ^([0-9]+)\.([0-9]+) ]]; then + BRANCH_VERSION="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" +else + BRANCH_VERSION="" +fi +if [[ -z "${BRANCH_VERSION}" ]]; then + echo "::error::Could not parse major.minor from version '${VERSION}'." + exit 1 +fi + +TARGET_BRANCH="release/${BRANCH_VERSION}" +CHERRY_PICK_BRANCH="dev/automation/pr-${PR_NUMBER}-to-${VERSION}" + +echo "Version: ${VERSION}" +echo "Target branch: ${TARGET_BRANCH}" +echo "Cherry-pick branch: ${CHERRY_PICK_BRANCH}" +echo "Merge commit: ${MERGE_COMMIT_SHA}" + +# Ensure the target branch ref is available locally. +git fetch origin "${TARGET_BRANCH}" + +# -- Step 2: Check if the patch is already applied ---------------------------- +# 'git cherry ' lists commits in .. and +# marks each with '-' (patch already on ) or '+' (not yet applied). +# By passing =MERGE_COMMIT_SHA and =MERGE_COMMIT_SHA^ we scope +# the check to exactly one commit — the PR being cherry-picked. +if git cherry "origin/${TARGET_BRANCH}" "${MERGE_COMMIT_SHA}" "${MERGE_COMMIT_SHA}^" \ + | grep -q '^-'; then + echo "::notice::Commit ${MERGE_COMMIT_SHA} is already applied on" \ + "${TARGET_BRANCH}. Skipping cherry-pick." + exit 0 +fi + +# Create the cherry-pick working branch from the target release branch. +git checkout -b "${CHERRY_PICK_BRANCH}" "origin/${TARGET_BRANCH}" + +# -- Step 3: Detect merge commit type ----------------------------------------- +# True merge commits have 2+ parents and require '--mainline 1' to tell git +# which parent's tree to diff against (the first parent = the target branch). +# Squash-merge commits have exactly 1 parent and must NOT use --mainline. +# +# 'git rev-list --parents -n1 ' outputs: [ ...] +# awk counts fields and subtracts 1 (the SHA itself) to get the parent count. +PARENT_COUNT=$(git rev-list --parents -n1 "${MERGE_COMMIT_SHA}" \ + | awk '{print NF - 1}') +MAINLINE_FLAG="" +if [[ "${PARENT_COUNT}" -gt 1 ]]; then + MAINLINE_FLAG="--mainline 1" + echo "Merge commit has ${PARENT_COUNT} parents — using --mainline 1." +else + echo "Squash-merge commit (single parent) — no --mainline flag needed." +fi + +# -- Helper: look up milestone ------------------------------------------------ +# Milestone assignment is best-effort. If the milestone doesn't exist yet, the +# PR is created without one and a note is appended to the PR body. +lookup_milestone() { + local version="$1" + MILESTONE_ARG="" + MILESTONE_NOTE="" + + if gh api "repos/${GITHUB_REPOSITORY}/milestones" --method GET --paginate \ + --field state=open --jq '.[].title' | grep -qx "${version}"; then + MILESTONE_ARG="--milestone ${version}" + echo "Milestone '${version}' found." + else + echo "::warning::Milestone '${version}' does not exist." \ + "PR will be created without a milestone." + MILESTONE_NOTE=$'\n\n> **Note:** Milestone `'"${version}"'` does not exist yet. Please create it and assign this PR manually.' + fi +} + +# -- Step 4: Attempt the cherry-pick ------------------------------------------ +# Options (--mainline) must precede the commit operand. +if git cherry-pick ${MAINLINE_FLAG} "${MERGE_COMMIT_SHA}"; then + # --- Success path --- + echo "Cherry-pick succeeded. Pushing branch and creating PR." + git push origin "${CHERRY_PICK_BRANCH}" + + lookup_milestone "${VERSION}" + + gh pr create \ + --base "${TARGET_BRANCH}" \ + --head "${CHERRY_PICK_BRANCH}" \ + --title "[${VERSION} Cherry-pick] ${PR_TITLE}" \ + --body "Cherry-pick of #${PR_NUMBER} (${MERGE_COMMIT_SHA}) into \`${TARGET_BRANCH}\`.${MILESTONE_NOTE}" \ + ${MILESTONE_ARG} +else + # --- Conflict path --- + echo "::error::Cherry-pick of ${MERGE_COMMIT_SHA} failed due to conflicts." + git cherry-pick --abort + + # Build the cherry-pick command for inclusion in the conflict-resolution + # instructions. Options must precede the commit SHA. + CHERRY_PICK_CMD="git cherry-pick" + if [[ -n "${MAINLINE_FLAG}" ]]; then + CHERRY_PICK_CMD="${CHERRY_PICK_CMD} ${MAINLINE_FLAG}" + fi + CHERRY_PICK_CMD="${CHERRY_PICK_CMD} ${MERGE_COMMIT_SHA}" + + # Create a branch with an empty commit so a PR can be opened. The PR body + # contains step-by-step instructions for manual conflict resolution. + git checkout "origin/${TARGET_BRANCH}" + git checkout -B "${CHERRY_PICK_BRANCH}" + git commit --allow-empty \ + -m "Cherry-pick of #${PR_NUMBER} requires manual resolution" \ + -m "To resolve, run: ${CHERRY_PICK_CMD}" + git push origin "${CHERRY_PICK_BRANCH}" + + lookup_milestone "${VERSION}" + + # Build the PR body using printf to avoid quoting pitfalls with embedded + # newlines (mixed $'...' and '...' quoting can leave literal \n in output). + CONFLICT_BODY="$(printf '%s' \ + "Cherry-pick of #${PR_NUMBER} (${MERGE_COMMIT_SHA}) into " \ + "\`${TARGET_BRANCH}\` **failed due to merge conflicts**." \ + "${MILESTONE_NOTE}" \ + $'\n\nPlease resolve manually:\n```bash\n' \ + "git fetch origin" \ + $'\n' \ + "git checkout ${CHERRY_PICK_BRANCH}" \ + $'\n' \ + "${CHERRY_PICK_CMD}" \ + $'\n' \ + "# resolve conflicts" \ + $'\n' \ + "git push origin ${CHERRY_PICK_BRANCH} --force" \ + $'\n```')" + + gh pr create \ + --draft \ + --base "${TARGET_BRANCH}" \ + --head "${CHERRY_PICK_BRANCH}" \ + --title "[${VERSION} Cherry-pick - CONFLICTS] ${PR_TITLE}" \ + ${MILESTONE_ARG} \ + --body "${CONFLICT_BODY}" +fi diff --git a/.github/scripts/extract-hotfix-versions.sh b/.github/scripts/extract-hotfix-versions.sh new file mode 100755 index 0000000000..b7c702a8f8 --- /dev/null +++ b/.github/scripts/extract-hotfix-versions.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# extract-hotfix-versions.sh +# +# Parses "Hotfix X.Y.Z" labels from a merged GitHub PR and emits a JSON array +# of version strings suitable for use as a GitHub Actions matrix dimension. +# +# OVERVIEW +# -------- +# This script handles two distinct trigger scenarios: +# +# 1. 'closed' event — The PR was just merged. ALL "Hotfix X.Y.Z" labels on +# the PR are processed, emitting one version per valid label. +# +# 2. 'labeled' event — A label was added to an already-merged PR. Only the +# NEWLY ADDED label is considered, and only if a cherry-pick for that +# version hasn't already been created (branch or PR exists). +# +# Label names must match the exact pattern "Hotfix .." +# (e.g. "Hotfix 7.0.1"). All other labels are silently ignored. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# LABELS Comma-separated list of all label names on the PR. +# EVENT_ACTION The GitHub event action: "closed" or "labeled". +# EVENT_LABEL For 'labeled' events, the name of the label that was added. +# Empty or unset for 'closed' events. +# PR_NUMBER The pull request number (used to derive cherry-pick branch names). +# GH_TOKEN GitHub token for API calls (gh CLI auth). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# +# OUTPUTS +# ------- +# Writes to $GITHUB_OUTPUT: +# versions= e.g. versions=["7.0.1","8.0.0"] +# +# An empty array (versions=[]) means no work is needed. +# The script exits with code 1 if the 'closed' event has no valid labels. +# +# USAGE +# Called from the cherry-pick-hotfix.yml workflow. Can also be run locally +# for testing by setting the required environment variables and providing a +# writable GITHUB_OUTPUT file: +# +# export LABELS="Hotfix 7.0.1,bug" +# export EVENT_ACTION="closed" +# export PR_NUMBER=42 +# export GITHUB_OUTPUT=$(mktemp) +# bash .github/scripts/extract-hotfix-versions.sh +# cat "$GITHUB_OUTPUT" +# +################################################################################# +set -euo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${LABELS:?LABELS environment variable is required}" +: "${EVENT_ACTION:?EVENT_ACTION environment variable is required}" +: "${PR_NUMBER:?PR_NUMBER environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +# -- 'labeled' event: process only the newly added label ---------------------- +if [[ "${EVENT_ACTION}" == "labeled" ]]; then + # Extract version from the new label. If it doesn't match "Hotfix X.Y.Z", + # this is a non-hotfix label — emit empty matrix and exit cleanly. + if [[ "${EVENT_LABEL:-}" =~ ^Hotfix\ ([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + CANDIDATE="${BASH_REMATCH[1]}" + else + CANDIDATE="" + fi + + if [[ -z "${CANDIDATE}" ]]; then + echo "Label '${EVENT_LABEL:-}' is not a valid 'Hotfix X.Y.Z' label. Skipping." + echo "versions=[]" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + # Guard against duplicate cherry-picks. If the cherry-pick branch already + # exists on the remote, or a PR (open, closed, or merged) was already created + # from it, there is nothing left to do. + # + # NOTE: We use the GitHub API rather than 'git ls-remote' because the + # detect-versions job does not check out the repository (no .git directory). + CHERRY_PICK_BRANCH="dev/automation/pr-${PR_NUMBER}-to-${CANDIDATE}" + + if gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/${CHERRY_PICK_BRANCH}" \ + --silent 2>/dev/null; then + echo "Cherry-pick branch '${CHERRY_PICK_BRANCH}' already exists. Skipping." + echo "versions=[]" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + EXISTING_PR=$(gh pr list --repo "${GITHUB_REPOSITORY}" --head "${CHERRY_PICK_BRANCH}" --state all \ + --json number --jq 'length') + if [[ "${EXISTING_PR}" -gt 0 ]]; then + echo "A cherry-pick PR from '${CHERRY_PICK_BRANCH}' already exists. Skipping." + echo "versions=[]" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + VERSIONS="${CANDIDATE}" +else + # -- 'closed' event: process all hotfix labels on the PR -------------------- + # Split by comma, keep only labels matching "Hotfix X.Y.Z", extract the version. + # Use sed -E for portable extended regex (works on both GNU and BSD sed). + VERSIONS=$(echo "${LABELS}" | tr ',' '\n' \ + | sed -nE 's/^Hotfix ([0-9]+\.[0-9]+\.[0-9]+)$/\1/p') +fi + +# -- Validate that at least one version was found ---------------------------- +if [[ -z "${VERSIONS}" ]]; then + echo "::error::No valid 'Hotfix X.Y.Z' label found. " \ + "Labels must match 'Hotfix ..'." + exit 1 +fi + +# -- Emit JSON array for the matrix strategy ---------------------------------- +# Convert the newline-separated version list into a compact JSON array. +# e.g. "7.0.1\n8.0.0" → ["7.0.1","8.0.0"] +JSON=$(echo "${VERSIONS}" \ + | jq -R -s -c 'split("\n") | map(select(length > 0))') + +echo "versions=${JSON}" >> "${GITHUB_OUTPUT}" +echo "Detected hotfix versions: ${JSON}" diff --git a/.github/scripts/recheck-milestones-for-release-branch.sh b/.github/scripts/recheck-milestones-for-release-branch.sh new file mode 100755 index 0000000000..b48ceb4ad6 --- /dev/null +++ b/.github/scripts/recheck-milestones-for-release-branch.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# recheck-milestones-for-release-branch.sh +# +# Re-runs the milestone check for open pull requests whose result can change +# when a release branch is created. +# +# OVERVIEW +# -------- +# check-milestone-branch.sh decides where a milestone belongs by asking whether +# release/. exists and which configured milestone series is the +# earliest one without a release branch. Cutting a branch therefore moves its +# X.Y.* milestones into servicing and can make the next series active. +# +# Creating a branch emits no pull request activity, so an already-open PR keeps +# whatever result it last recorded. This script closes that gap by re-running +# every semantic-version-milestoned PR targeting the default branch. That covers +# both the newly serviced series and any later series whose eligibility changes. +# +# Re-running is enough because check-milestone-branch.sh queries the live list +# of release branches. The replayed event payload still carries the correct +# milestone and base branch, since the check re-runs on every 'milestoned' and +# 'edited' activity, so the newest run always reflects the current PR state. +# +# LIMITATION +# ---------- +# A re-run replays the original run's commit, which means it executes the +# workflow definition and script as they existed then. Only the release branch +# lookup is evaluated live. +# +# So a pull request whose most recent milestone check predates the arrival of +# the target-branch rule will replay the older check, pass it, and keep its +# stale result. This is transitional: any pull request with activity after the +# rule shipped has a run that contains it. It is not detected here, because +# distinguishing a stale replay costs an extra API call per pull request and +# the window closes on its own. After the first release branch is cut following +# a change to the check itself, review the affected pull requests by hand. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# RELEASE_BRANCH The branch that was just created (e.g. "release/7.1"). +# DEFAULT_BRANCH The repository's default branch (e.g. "main"). +# WORKFLOW_FILE Workflow file name to re-run (e.g. "check-milestone.yml"). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# GH_TOKEN GitHub token for API calls. Needs 'actions: write'. +# +# OUTPUTS +# ------- +# Emits ::notice:: per PR re-run and ::warning:: for any PR that could not be +# re-run. Exits 1 if at least one re-run failed, so the failure is visible in +# the Actions UI; a maintainer can then re-run those checks by hand. +# +# USAGE +# Called from the recheck-milestones.yml workflow. Can also be run locally: +# +# export RELEASE_BRANCH="release/7.1" +# export DEFAULT_BRANCH="main" +# export WORKFLOW_FILE="check-milestone.yml" +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/recheck-milestones-for-release-branch.sh +# +################################################################################# +# 'set -e' is deliberately omitted: one PR failing to re-run must not abandon +# the rest. Failures are tracked explicitly and reported at the end. +set -uo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${RELEASE_BRANCH:?RELEASE_BRANCH environment variable is required}" +: "${DEFAULT_BRANCH:?DEFAULT_BRANCH environment variable is required}" +: "${WORKFLOW_FILE:?WORKFLOW_FILE environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +if [[ ! "${RELEASE_BRANCH}" =~ ^release/[0-9]+\.[0-9]+$ ]]; then + echo "::notice::'${RELEASE_BRANCH}' is not a 'release/.' branch; nothing to reconcile." + exit 0 +fi + +# -- Find open PRs whose result can change ------------------------------------- +if ! OPEN_PRS=$(gh pr list --repo "${GITHUB_REPOSITORY}" --base "${DEFAULT_BRANCH}" \ + --state open --limit 500 --json number,headRefOid,milestone \ + --jq '.[] | select(.milestone != null) | "\(.number) \(.headRefOid) \(.milestone.title)"' 2>&1); then + echo "::error::Unable to list open pull requests for '${GITHUB_REPOSITORY}': ${OPEN_PRS}" + exit 1 +fi + +FAILED=0 +MATCHED=0 + +while read -r NUMBER HEAD_SHA MILESTONE_TITLE; do + [[ -n "${NUMBER}" ]] || continue + + # Same milestone grammar as check-milestone-branch.sh. + [[ "${MILESTONE_TITLE}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+].*)?$ ]] || continue + + MATCHED=$((MATCHED + 1)) + + RUNS=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?head_sha=${HEAD_SHA}&per_page=100" \ + 2>/dev/null) + + # One head commit can back PRs against several bases, so prefer the run that + # names this PR rather than just the newest run for the SHA. + RUN_ID=$(jq -r --argjson pr "${NUMBER}" \ + '([.workflow_runs[] | select(any(.pull_requests[]?; .number == $pr))] | first | .id) // empty' \ + <<< "${RUNS}" 2>/dev/null) + + # 'pull_requests' is empty for runs from forked repositories, so fall back to + # the newest run for the SHA when the association is unavailable. + if [[ -z "${RUN_ID}" ]]; then + RUN_ID=$(jq -r '.workflow_runs[0].id // empty' <<< "${RUNS}" 2>/dev/null) + fi + + if [[ -z "${RUN_ID}" ]]; then + echo "::warning::PR #${NUMBER} (milestone '${MILESTONE_TITLE}') has no milestone check run to re-run; re-check it manually." + FAILED=$((FAILED + 1)) + continue + fi + + if gh run rerun "${RUN_ID}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "::notice::Re-ran the milestone check for PR #${NUMBER} (milestone '${MILESTONE_TITLE}', run ${RUN_ID})." + else + echo "::warning::Could not re-run the milestone check for PR #${NUMBER} (run ${RUN_ID}); re-check it manually." + FAILED=$((FAILED + 1)) + fi +done <<< "${OPEN_PRS}" + +if [[ "${MATCHED}" -eq 0 ]]; then + echo "::notice::No open PR targeting '${DEFAULT_BRANCH}' carries a semantic-version milestone." + exit 0 +fi + +if [[ "${FAILED}" -gt 0 ]]; then + echo "::error::${FAILED} of ${MATCHED} affected pull requests could not be re-checked automatically." + exit 1 +fi + +echo "::notice::Re-checked ${MATCHED} pull request(s) affected by '${RELEASE_BRANCH}'." diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md new file mode 100644 index 0000000000..9a4ea68906 --- /dev/null +++ b/.github/scripts/tests/README.md @@ -0,0 +1,182 @@ +# GitHub Actions Script Tests + +This directory contains tests for the shell scripts used by the +[cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml), +[check-milestone](./../../../.github/workflows/check-milestone.yml) and +[recheck-milestones](./../../../.github/workflows/recheck-milestones.yml) GitHub Actions workflows. +These tests are intended to be run manually by developers when they are changing the associated +scripts, and not as part of any CI runs. + +## What is Bats? + +**[Bats](https://github.com/bats-core/bats-core)** (Bash Automated Testing System) is a +TAP-compliant testing framework for Bash scripts. Each `.bats` file contains one or more `@test` +blocks that run shell commands and assert outcomes using the built-in `run` helper. A test passes +when every command exits with code 0; it fails on the first non-zero exit. + +Key concepts: + +| Concept | Description | +| ------- | ----------- | +| `@test "name" { ... }` | Defines a single test case | +| `run ` | Captures stdout, stderr, and exit code into `$output` and `$status` | +| `setup()` | Runs before every `@test` — used to create temp files and set env vars | +| `teardown()` | Runs after every `@test` — used to clean up temp files | +| `[[ "$status" -eq 0 ]]` | Assert exit code | +| `[[ "$output" == *"text"* ]]` | Assert output contains a string | + +## Installing Bats + +### Linux (apt) + +```bash +sudo apt-get update && sudo apt-get install -y bats +``` + +### macOS (Homebrew) + +```bash +brew install bats-core +``` + +### From source (any platform) + +```bash +git clone https://github.com/bats-core/bats-core.git +cd bats-core +sudo ./install.sh /usr/local +``` + +## Additional Prerequisites + +The scripts under test also require: + +- **jq** — used to build JSON matrix output +- **gh** (GitHub CLI) — used for API calls and PR creation (mocked during tests, but must be on + `$PATH` for the test stubs to shadow it) + +Most CI runners and development machines have these pre-installed. If not: + +```bash +# Linux (apt) +sudo apt-get install -y jq gh + +# macOS (Homebrew) +brew install jq gh +``` + +### Verify installation + +```bash +bats --version +# Expected output: Bats 1.x.x +``` + +## Running the Tests + +All commands assume you are at the **repository root**. + +### Run all tests + +```bash +bats .github/scripts/tests/ +``` + +### Run a single test file + +```bash +bats .github/scripts/tests/extract-hotfix-versions.bats +bats .github/scripts/tests/cherry-pick-to-release.bats +bats .github/scripts/tests/check-milestone-branch.bats +bats .github/scripts/tests/recheck-milestones-for-release-branch.bats +``` + +### Run a specific test by name + +```bash +bats .github/scripts/tests/extract-hotfix-versions.bats \ + --filter "single Hotfix label" +``` + +### Verbose output (show each test name) + +```bash +bats --tap .github/scripts/tests/ +``` + +### Pretty output (requires bats-core 1.5+) + +```bash +bats --formatter pretty .github/scripts/tests/ +``` + +## Test Files + +| File | Tests | Covers | +| ---- | ----- | ------ | +| `extract-hotfix-versions.bats` | 18 | Label parsing, version extraction, matrix JSON output, edge cases (malformed labels, duplicates, `labeled` vs `closed` events) | +| `cherry-pick-to-release.bats` | 15 | Branch derivation, already-applied detection, clean cherry-pick, conflict handling, milestone lookup, PR creation, duplicate skip logic | +| `check-milestone-branch.bats` | 26 | Milestone version parsing, state-independent active development-line selection, rejection of earlier and later series on the default branch, fail-closed handling when no series is active, release-branch derivation, default-branch vs release-branch validation, integration-branch and non-semver skips, API invocation assertions, API failure handling | +| `recheck-milestones-for-release-branch.bats` | 17 | Release-branch name parsing, matching open PRs by milestone, run lookup by head SHA and PR association, fork fallback, re-run invocation, and failure reporting | + +## How the Tests Work + +All test files use the same general approach: + +1. **`setup()`** creates a temporary directory and populates it with mock `git` and `gh` executables + — simple shell scripts that echo predetermined responses. Environment variables (`VERSION`, + `MERGE_COMMIT_SHA`, etc.) are set to known values. + +2. **`@test` blocks** call `run bash "$SCRIPT"` to execute the script under test in a subshell. The + mocks intercept all `git` and `gh` invocations, so no real repository or GitHub API access is + needed. + +3. **Assertions** check `$status` (exit code) and `$output` (combined stdout/stderr) for expected + values, error messages, GitHub Actions output file writes via `$GITHUB_OUTPUT`, or other + workflow commands such as `::error::` and `::notice::`. + +4. **`teardown()`** removes the temporary directory and mock binaries. + +### Example mock + +```bash +# Mock git that reports 2 parents (a merge commit) +cat > "${STUB_DIR}/git" <<'MOCK' +#!/usr/bin/env bash +case "$*" in + "rev-list --parents -n1 "*) echo "abc123 parent1 parent2" ;; + "cherry "*) echo "+ abc123" ;; + *) echo "git mock: $*" ;; +esac +MOCK +chmod +x "${STUB_DIR}/git" +``` + +The mock sits earlier on `$PATH` than the real `git`, so the script under test calls the mock +transparently. + +## Troubleshooting + +### `bats: command not found` + +Bats is not installed. See [Installing Bats](#installing-bats) above. + +### Tests fail with `permission denied` + +The scripts under `.github/scripts/` must be executable: + +```bash +chmod +x .github/scripts/*.sh +``` + +### A test fails unexpectedly + +Run with `set -x` tracing to see each command: + +```bash +bats --tap .github/scripts/tests/cherry-pick-to-release.bats \ + --filter "name of failing test" 2>&1 +``` + +Or add `echo "DEBUG: $variable" >&3` inside a test to print to the terminal (file descriptor 3 is +bats's "direct to terminal" channel). diff --git a/.github/scripts/tests/check-milestone-branch.bats b/.github/scripts/tests/check-milestone-branch.bats new file mode 100644 index 0000000000..813605a012 --- /dev/null +++ b/.github/scripts/tests/check-milestone-branch.bats @@ -0,0 +1,317 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for check-milestone-branch.sh +# +# Run with: bats .github/scripts/tests/check-milestone-branch.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/check-milestone-branch.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + export DEFAULT_BRANCH="main" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GH_TOKEN="fake-token" + export MOCK_MILESTONES=$'1.0.0\n2.0.1\n7.1.0\n8.0.0-preview1\n8.0.0-preview2\n8.0.0' + + mock_release_branches "release/6.1" "release/7.0" +} + +teardown() { + rm -rf "${STUB_DIR}" +} + +# Install a 'gh' mock that reports the given release branches. +mock_release_branches() { + local refs="" + local branch + for branch in "$@"; do + refs+="refs/heads/${branch}"$'\n' + done + + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +if [[ "\$*" == *"/milestones"* ]]; then + printf '%s' "\${MOCK_MILESTONES}" +else + printf '%s' '${refs}' +fi +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock that fails, simulating an API error. +mock_gh_failure() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +echo "HTTP 403: rate limit exceeded" >&2 +exit 1 +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock that lists branches but fails when milestones are queried. +mock_milestone_failure() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +if [[ "\$*" == *"/milestones"* ]]; then + echo "HTTP 403: resource not accessible by integration" >&2 + exit 1 +fi +printf '%s' 'refs/heads/release/6.1 +refs/heads/release/7.0 +' +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"VALIDATION MATRIX"* ]] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +@test "prints help text with -h" { + run bash "${SCRIPT}" -h + [ "$status" -eq 0 ] + [[ "$output" == *"VALIDATION MATRIX"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when MILESTONE_TITLE is unset" { + unset MILESTONE_TITLE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"MILESTONE_TITLE"* ]] +} + +@test "fails when BASE_REF is unset" { + unset BASE_REF + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"BASE_REF"* ]] +} + +@test "fails when DEFAULT_BRANCH is unset" { + unset DEFAULT_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"DEFAULT_BRANCH"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Default branch targets ─────────────────────────────────────────────────── + +@test "passes when in-development milestone targets the default branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"::notice::"* ]] + [[ "$output" == *"still in development"* ]] +} + +@test "fails when a later milestone targets the default branch before the active release branch is cut" { + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"7.1 milestone series"* ]] + [[ "$output" == *"release/7.1"* ]] +} + +@test "closed milestone state does not change the active development line" { + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"7.1 milestone series"* ]] + [[ "$output" != *"1.0.0"* ]] + grep -qF "milestones?state=all" "${STUB_DIR}/gh.log" +} + +@test "fails when an earlier milestone series targets the default branch" { + export MILESTONE_TITLE="1.0.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"no longer in development"* ]] + [[ "$output" == *"active line is 7.1"* ]] +} + +@test "fails when no configured milestone series is active" { + mock_release_branches "release/7.0" + export MOCK_MILESTONES=$'1.0.0' + export MILESTONE_TITLE="1.0.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"no development line is active"* ]] +} + +@test "passes when a later milestone targets the default branch after the active release branch is cut" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"still in development"* ]] +} + +@test "fails when a serviced milestone targets the default branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" == *"release/7.0"* ]] + [[ "$output" == *"Hotfix 7.0.3"* ]] +} + +@test "honours a non-'main' default branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="master" + export DEFAULT_BRANCH="master" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"targeting 'master' is correct"* ]] +} + +# ── API invocation ─────────────────────────────────────────────────────────── + +@test "queries the release refs endpoint with the expected arguments" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: api repos/dotnet/SqlClient/git/matching-refs/heads/release/ --jq .[].ref" "${STUB_DIR}/gh.log" + grep -qF "GH: api --paginate repos/dotnet/SqlClient/milestones?state=all&per_page=100 --jq .[].title" "${STUB_DIR}/gh.log" +} + +@test "does not call the API when the PR targets a release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ ! -f "${STUB_DIR}/gh.log" ] +} + +# ── Release branch targets ─────────────────────────────────────────────────── + +@test "passes when the milestone matches the target release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"matches target branch 'release/7.0'"* ]] +} + +@test "fails when the milestone belongs to a different release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/6.1" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" == *"belongs to 'release/7.0'"* ]] +} + +@test "fails when an in-development milestone targets a release branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"belongs to 'release/7.1'"* ]] +} + +@test "passes when a newly cut release branch matches the milestone" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="7.1.0" + export BASE_REF="release/7.1" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"matches target branch 'release/7.1'"* ]] +} + +@test "fails on the default branch once the release branch is cut" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"release/7.1"* ]] +} + +# ── Skipped cases ──────────────────────────────────────────────────────────── + +@test "skips integration branch targets" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="dev/paul/some-feature" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"integration branch"* ]] +} + +@test "skips milestones that are not major.minor.patch" { + export MILESTONE_TITLE="vNext" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"not in 'major.minor.patch' form"* ]] +} + +@test "skips two-part milestone titles" { + export MILESTONE_TITLE="1.0 Hotfix 2" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"not in 'major.minor.patch' form"* ]] +} + +# ── API failures ───────────────────────────────────────────────────────────── + +@test "fails when the branch listing API call fails" { + mock_gh_failure + export MILESTONE_TITLE="7.0.3" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list release branches"* ]] +} + +@test "fails when the milestone listing API call fails" { + mock_milestone_failure + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list milestones"* ]] +} diff --git a/.github/scripts/tests/cherry-pick-to-release.bats b/.github/scripts/tests/cherry-pick-to-release.bats new file mode 100644 index 0000000000..3484b48ea9 --- /dev/null +++ b/.github/scripts/tests/cherry-pick-to-release.bats @@ -0,0 +1,347 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for cherry-pick-to-release.sh +# +# Run with: bats .github/scripts/tests/cherry-pick-to-release.bats +# +# NOTE: These tests mock git and gh commands to validate the script's logic +# without requiring a real repository or GitHub API access. +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/cherry-pick-to-release.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + # Create a directory for mock binaries that override real git/gh. + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export VERSION="7.0.1" + export MERGE_COMMIT_SHA="abc123def456" + export PR_NUMBER="42" + export PR_TITLE="Fix connection timeout" + export GH_TOKEN="fake-token" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GITHUB_OUTPUT="$(mktemp)" +} + +teardown() { + rm -rf "${STUB_DIR}" + rm -f "${GITHUB_OUTPUT}" +} + +# Write a mock 'git' script. Each call to the mock appends a log line so +# tests can verify which git subcommands were executed and with what args. +write_git_mock() { + local body="$1" + cat > "${STUB_DIR}/git" <> "${STUB_DIR}/git.log" +${body} +STUB + chmod +x "${STUB_DIR}/git" +} + +# Write a mock 'gh' script. +write_gh_mock() { + local body="$1" + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +${body} +STUB + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"Cherry-picks a merge commit"* ]] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +@test "prints help text with -h" { + run bash "${SCRIPT}" -h + [ "$status" -eq 0 ] + [[ "$output" == *"Cherry-picks"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when VERSION is unset" { + unset VERSION + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"VERSION"* ]] +} + +@test "fails when MERGE_COMMIT_SHA is unset" { + unset MERGE_COMMIT_SHA + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"MERGE_COMMIT_SHA"* ]] +} + +@test "fails when PR_NUMBER is unset" { + unset PR_NUMBER + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"PR_NUMBER"* ]] +} + +@test "fails when PR_TITLE is unset" { + unset PR_TITLE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"PR_TITLE"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Version parsing ───────────────────────────────────────────────────────── + +@test "derives release/7.0 from version 7.0.1" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + if [[ "$1" == "rev-list" ]]; then echo "abc123 parent1"; exit 0; fi + if [[ "$1" == "cherry-pick" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + exit 0 + ' + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "7.0.1"; exit 0; fi + if [[ "$1" == "pr" ]]; then exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + # Verify the git fetch targeted release/7.0. + grep -q "GIT: fetch origin release/7.0" "${STUB_DIR}/git.log" + # Milestone lookup must use GET to avoid accidentally POSTing to the create endpoint. + grep "GH: api repos/dotnet/SqlClient/milestones" "${STUB_DIR}/gh.log" | grep -q "\-\-method GET" +} + +@test "derives release/8.0 from version 8.0.0" { + export VERSION="8.0.0" + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + if [[ "$1" == "rev-list" ]]; then echo "abc123 parent1"; exit 0; fi + if [[ "$1" == "cherry-pick" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + exit 0 + ' + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "8.0.0"; exit 0; fi + if [[ "$1" == "pr" ]]; then exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -q "GIT: fetch origin release/8.0" "${STUB_DIR}/git.log" +} + +@test "fails on unparseable version" { + export VERSION="bad" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Could not parse"* ]] +} + +# ── Already-applied detection ─────────────────────────────────────────────── + +@test "exits cleanly when commit is already applied" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + # git cherry: "-" prefix means patch is already applied. + if [[ "$1" == "cherry" ]]; then echo "- abc123def456"; exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"already applied"* ]] + # Should NOT have attempted a cherry-pick. + ! grep -q "GIT: cherry-pick" "${STUB_DIR}/git.log" +} + +# ── Squash-merge detection (single parent) ────────────────────────────────── + +@test "does not use --mainline for squash merges" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + # Single parent: rev-list outputs "sha parent1" (2 fields → 1 parent). + if [[ "$1" == "rev-list" ]]; then echo "abc123def456 parent1"; exit 0; fi + if [[ "$1" == "cherry-pick" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + exit 0 + ' + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "7.0.1"; exit 0; fi + if [[ "$1" == "pr" ]]; then exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"single parent"* ]] + # cherry-pick should NOT include --mainline. + grep "GIT: cherry-pick" "${STUB_DIR}/git.log" | grep -qv "\-\-mainline" +} + +# ── True merge detection (multiple parents) ───────────────────────────────── + +@test "uses --mainline 1 for true merge commits" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + # Two parents: rev-list outputs "sha parent1 parent2" (3 fields → 2 parents). + if [[ "$1" == "rev-list" ]]; then echo "abc123def456 parent1 parent2"; exit 0; fi + if [[ "$1" == "cherry-pick" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + exit 0 + ' + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "7.0.1"; exit 0; fi + if [[ "$1" == "pr" ]]; then exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"--mainline 1"* ]] +} + +# ── Milestone lookup ──────────────────────────────────────────────────────── + +@test "warns when milestone does not exist" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + if [[ "$1" == "rev-list" ]]; then echo "abc123def456 parent1"; exit 0; fi + if [[ "$1" == "cherry-pick" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + if [[ "$1" == "config" ]]; then exit 0; fi + exit 0 + ' + # gh api returns a milestone that does NOT match VERSION (7.0.1). + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "6.0.0"; exit 0; fi + if [[ "$1" == "pr" ]]; then exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"does not exist"* ]] + # Milestone lookup must use GET to avoid accidentally POSTing to the create endpoint. + grep "GH: api repos/dotnet/SqlClient/milestones" "${STUB_DIR}/gh.log" | grep -q "\-\-method GET" +} + +# ── Conflict handling ─────────────────────────────────────────────────────── + +@test "creates CONFLICTS PR when cherry-pick fails" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + if [[ "$1" == "rev-list" ]]; then echo "abc123def456 parent1"; exit 0; fi + # cherry-pick fails with conflicts. + if [[ "$1" == "cherry-pick" ]]; then + if [[ "$2" == "--abort" ]]; then exit 0; fi + exit 1 + fi + if [[ "$1" == "commit" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + exit 0 + ' + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "7.0.1"; exit 0; fi + if [[ "$1" == "pr" ]]; then exit 0; fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"failed due to conflicts"* ]] + # Should have called cherry-pick --abort. + grep -q "GIT: cherry-pick --abort" "${STUB_DIR}/git.log" + # Should have created an empty commit. + grep -q "GIT: commit --allow-empty" "${STUB_DIR}/git.log" +} + +@test "conflict PR body contains real newlines, not literal backslash-n" { + write_git_mock ' + if [[ "$1" == "fetch" ]]; then exit 0; fi + if [[ "$1" == "cherry" ]]; then echo "+ abc123"; exit 0; fi + if [[ "$1" == "checkout" ]]; then exit 0; fi + if [[ "$1" == "rev-list" ]]; then echo "abc123def456 parent1"; exit 0; fi + if [[ "$1" == "cherry-pick" ]]; then + if [[ "$2" == "--abort" ]]; then exit 0; fi + exit 1 + fi + if [[ "$1" == "commit" ]]; then exit 0; fi + if [[ "$1" == "push" ]]; then exit 0; fi + exit 0 + ' + # Capture the full --body argument to a file for inspection. + write_gh_mock ' + if [[ "$1" == "api" ]]; then echo "7.0.1"; exit 0; fi + if [[ "$1" == "pr" && "$2" == "create" ]]; then + while [[ $# -gt 0 ]]; do + if [[ "$1" == "--body" ]]; then + printf "%s" "$2" > "'"${STUB_DIR}"'/pr-body.txt" + break + fi + shift + done + exit 0 + fi + exit 0 + ' + + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + + # The body file must exist (gh pr create was called with --body). + [ -f "${STUB_DIR}/pr-body.txt" ] + + local body + body="$(cat "${STUB_DIR}/pr-body.txt")" + + # Must NOT contain literal two-character sequence '\n'. + [[ "$body" != *'\\n'* ]] + # Each command in the code block must be on its own line. + [[ "$body" == *$'\ngit fetch origin\n'* ]] + [[ "$body" == *$'\ngit checkout dev/automation/pr-42-to-7.0.1\n'* ]] + [[ "$body" == *$'\ngit cherry-pick abc123def456\n'* ]] + [[ "$body" == *$'\n# resolve conflicts\n'* ]] + [[ "$body" == *$'\ngit push origin dev/automation/pr-42-to-7.0.1 --force\n'* ]] +} diff --git a/.github/scripts/tests/extract-hotfix-versions.bats b/.github/scripts/tests/extract-hotfix-versions.bats new file mode 100644 index 0000000000..e0767ffd04 --- /dev/null +++ b/.github/scripts/tests/extract-hotfix-versions.bats @@ -0,0 +1,244 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for extract-hotfix-versions.sh +# +# Run with: bats .github/scripts/tests/extract-hotfix-versions.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/extract-hotfix-versions.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + # Create a temporary GITHUB_OUTPUT file for each test. + export GITHUB_OUTPUT + GITHUB_OUTPUT="$(mktemp)" + + # Defaults — individual tests override as needed. + export EVENT_ACTION="closed" + export EVENT_LABEL="" + export PR_NUMBER="42" + export GH_TOKEN="fake-token" + export GITHUB_REPOSITORY="dotnet/SqlClient" +} + +teardown() { + rm -f "${GITHUB_OUTPUT}" +} + +# Read the 'versions' output written to GITHUB_OUTPUT. +get_versions() { + grep '^versions=' "${GITHUB_OUTPUT}" | head -1 | cut -d= -f2- +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"Parses"* ]] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +@test "prints help text with -h" { + run bash "${SCRIPT}" -h + [ "$status" -eq 0 ] + [[ "$output" == *"Parses"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when LABELS is unset" { + unset LABELS + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"LABELS"* ]] +} + +@test "fails when EVENT_ACTION is unset" { + export LABELS="Hotfix 7.0.1" + unset EVENT_ACTION + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"EVENT_ACTION"* ]] +} + +@test "fails when PR_NUMBER is unset" { + export LABELS="Hotfix 7.0.1" + unset PR_NUMBER + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"PR_NUMBER"* ]] +} + +# ── Closed event: single label ────────────────────────────────────────────── + +@test "closed event: extracts single Hotfix label" { + export LABELS="Hotfix 7.0.1" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ "$(get_versions)" = '["7.0.1"]' ] +} + +@test "closed event: ignores non-hotfix labels" { + export LABELS="bug,Hotfix 7.0.1,enhancement" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ "$(get_versions)" = '["7.0.1"]' ] +} + +# ── Closed event: multiple labels ─────────────────────────────────────────── + +@test "closed event: extracts multiple Hotfix labels" { + export LABELS="Hotfix 7.0.1,Hotfix 8.0.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ "$(get_versions)" = '["7.0.1","8.0.0"]' ] +} + +@test "closed event: extracts hotfix labels mixed with other labels" { + export LABELS="bug,Hotfix 7.0.1,enhancement,Hotfix 8.0.0,docs" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ "$(get_versions)" = '["7.0.1","8.0.0"]' ] +} + +# ── Closed event: malformed labels ────────────────────────────────────────── + +@test "closed event: rejects malformed Hotfix labels (no patch)" { + export LABELS="Hotfix 7.0" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"No valid"* ]] +} + +@test "closed event: rejects Hotfix label with text suffix" { + export LABELS="Hotfix 7.0.1-beta" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"No valid"* ]] +} + +@test "closed event: rejects Hotfix label with non-numeric version" { + export LABELS="Hotfix abc" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"No valid"* ]] +} + +@test "closed event: fails when no labels present" { + export LABELS="" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] +} + +# ── Labeled event: basic behavior ─────────────────────────────────────────── + +@test "labeled event: processes valid newly added label" { + export EVENT_ACTION="labeled" + export EVENT_LABEL="Hotfix 7.0.1" + export LABELS="Hotfix 7.0.1,Hotfix 8.0.0" + + # Mock gh to report no existing branch or PR. + # The script now uses 'gh api' for branch checks (no git checkout in this job). + local stub_dir + stub_dir="$(mktemp -d)" + cat > "${stub_dir}/gh" <<'STUB' +#!/usr/bin/env bash +# gh api repos/.../git/ref/heads/...: exit 1 (branch not found) +# gh pr list: return "0" PRs +if [[ "$1" == "api" && "$2" == repos/*/git/ref/heads/* ]]; then + exit 1 +fi +echo "0" +STUB + chmod +x "${stub_dir}/gh" + + export PATH="${stub_dir}:${PATH}" + run bash "${SCRIPT}" + rm -rf "${stub_dir}" + + [ "$status" -eq 0 ] + [ "$(get_versions)" = '["7.0.1"]' ] +} + +@test "labeled event: skips non-hotfix label" { + export EVENT_ACTION="labeled" + export EVENT_LABEL="bug" + export LABELS="bug,Hotfix 7.0.1" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ "$(get_versions)" = '[]' ] +} + +@test "labeled event: skips malformed hotfix label" { + export EVENT_ACTION="labeled" + export EVENT_LABEL="Hotfix 7.0" + export LABELS="Hotfix 7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ "$(get_versions)" = '[]' ] +} + +# ── Labeled event: duplicate detection ────────────────────────────────────── + +@test "labeled event: skips when cherry-pick branch already exists" { + export EVENT_ACTION="labeled" + export EVENT_LABEL="Hotfix 7.0.1" + export LABELS="Hotfix 7.0.1" + + local stub_dir + stub_dir="$(mktemp -d)" + # gh api returns success — branch exists on the remote. + cat > "${stub_dir}/gh" <<'STUB' +#!/usr/bin/env bash +if [[ "$1" == "api" && "$2" == repos/*/git/ref/heads/* ]]; then + exit 0 +fi +echo "0" +STUB + chmod +x "${stub_dir}/gh" + export PATH="${stub_dir}:${PATH}" + + run bash "${SCRIPT}" + rm -rf "${stub_dir}" + + [ "$status" -eq 0 ] + [ "$(get_versions)" = '[]' ] + [[ "$output" == *"already exists"* ]] +} + +@test "labeled event: skips when cherry-pick PR already exists" { + export EVENT_ACTION="labeled" + export EVENT_LABEL="Hotfix 7.0.1" + export LABELS="Hotfix 7.0.1" + + local stub_dir + stub_dir="$(mktemp -d)" + # gh api for branch check returns 1 (not found), but pr list returns 1 PR. + cat > "${stub_dir}/gh" <<'STUB' +#!/usr/bin/env bash +if [[ "$1" == "api" && "$2" == repos/*/git/ref/heads/* ]]; then + exit 1 +fi +echo "1" +STUB + chmod +x "${stub_dir}/gh" + export PATH="${stub_dir}:${PATH}" + + run bash "${SCRIPT}" + rm -rf "${stub_dir}" + + [ "$status" -eq 0 ] + [ "$(get_versions)" = '[]' ] + [[ "$output" == *"already exists"* ]] +} diff --git a/.github/scripts/tests/recheck-milestones-for-release-branch.bats b/.github/scripts/tests/recheck-milestones-for-release-branch.bats new file mode 100644 index 0000000000..0d48388f33 --- /dev/null +++ b/.github/scripts/tests/recheck-milestones-for-release-branch.bats @@ -0,0 +1,228 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for recheck-milestones-for-release-branch.sh +# +# Run with: bats .github/scripts/tests/recheck-milestones-for-release-branch.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/recheck-milestones-for-release-branch.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export RELEASE_BRANCH="release/7.1" + export DEFAULT_BRANCH="main" + export WORKFLOW_FILE="check-milestone.yml" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GH_TOKEN="fake-token" + + # Open PRs as " ", one per line. + mock_gh "100 aaa111 7.1.0 +101 bbb222 8.0.0-preview1 +102 ccc333 7.1.0-preview3" + + # Default: one run per head SHA, correctly associated with its PR. + set_runs aaa111 '{"workflow_runs":[{"id":"run-aaa111","pull_requests":[{"number":100}]}]}' + set_runs bbb222 '{"workflow_runs":[{"id":"run-bbb222","pull_requests":[{"number":101}]}]}' + set_runs ccc333 '{"workflow_runs":[{"id":"run-ccc333","pull_requests":[{"number":102}]}]}' +} + +teardown() { + rm -rf "${STUB_DIR}" +} + +# Register the workflow-runs response for a given head SHA. +set_runs() { + printf '%s' "$2" > "${STUB_DIR}/runs-$1.json" +} + +# Install a 'gh' mock. $1 is the 'pr list' output; 'api' serves the JSON +# registered by set_runs, and 'run rerun' succeeds unless RERUN_FAILS is set. +mock_gh() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +case "\$1" in + pr) + printf '%s\n' '${1}' + ;; + api) + sha="\$(sed -n 's/.*head_sha=\([^&]*\).*/\1/p' <<< "\$2")" + if [[ -n "\${NO_RUN_FOUND:-}" || ! -f "${STUB_DIR}/runs-\${sha}.json" ]]; then + echo '{"workflow_runs":[]}' + else + cat "${STUB_DIR}/runs-\${sha}.json" + fi + ;; + run) + [[ -z "\${RERUN_FAILS:-}" ]] || exit 1 + ;; +esac +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock whose 'pr list' call fails. +mock_pr_list_failure() { + cat > "${STUB_DIR}/gh" <<'MOCK' +#!/usr/bin/env bash +echo "HTTP 403: rate limit exceeded" >&2 +exit 1 +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when RELEASE_BRANCH is unset" { + unset RELEASE_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"RELEASE_BRANCH"* ]] +} + +@test "fails when DEFAULT_BRANCH is unset" { + unset DEFAULT_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"DEFAULT_BRANCH"* ]] +} + +@test "fails when WORKFLOW_FILE is unset" { + unset WORKFLOW_FILE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"WORKFLOW_FILE"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Branch name parsing ────────────────────────────────────────────────────── + +@test "skips branches that are not release/." { + export RELEASE_BRANCH="dev/paul/some-feature" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to reconcile"* ]] + [ ! -f "${STUB_DIR}/gh.log" ] +} + +@test "skips a release branch with a patch component" { + export RELEASE_BRANCH="release/7.1.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to reconcile"* ]] +} + +# ── Matching and re-running ────────────────────────────────────────────────── + +@test "re-runs all semver-milestoned PRs affected by active-line transitions" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"PR #100"* ]] + [[ "$output" == *"PR #102"* ]] + [[ "$output" == *"PR #101"* ]] + [[ "$output" == *"Re-checked 3 pull request(s)"* ]] +} + +@test "queries open PRs against the default branch" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: pr list --repo dotnet/SqlClient --base main --state open" "${STUB_DIR}/gh.log" +} + +@test "looks up the run by the PR head sha and re-runs it" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "head_sha=aaa111" "${STUB_DIR}/gh.log" + grep -qF "GH: run rerun run-aaa111 --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" +} + +@test "picks the run belonging to this PR when a head sha backs several PRs" { + # The newest run for the SHA belongs to a different PR against another base. + set_runs aaa111 '{"workflow_runs":[ + {"id":"run-other","pull_requests":[{"number":999}]}, + {"id":"run-aaa111","pull_requests":[{"number":100}]} + ]}' + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: run rerun run-aaa111 --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" + ! grep -qF "run rerun run-other" "${STUB_DIR}/gh.log" +} + +@test "falls back to the newest run when the PR association is missing" { + # Runs from forked repositories carry an empty 'pull_requests' array. + set_runs aaa111 '{"workflow_runs":[ + {"id":"run-newest","pull_requests":[]}, + {"id":"run-older","pull_requests":[]} + ]}' + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"PR #100"* ]] + grep -qF "GH: run rerun run-newest --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" +} + +@test "reports when no open PR carries a matching milestone" { + export RELEASE_BRANCH="release/6.1" + mock_gh "200 ddd444 vNext" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"No open PR targeting 'main' carries a semantic-version milestone"* ]] +} + +@test "ignores PRs whose milestone is not major.minor.patch" { + mock_gh "200 ddd444 vNext" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"No open PR"* ]] +} + +# ── Failure handling ───────────────────────────────────────────────────────── + +@test "warns and fails when a PR has no run to re-run" { + export NO_RUN_FOUND=1 + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"has no milestone check run to re-run"* ]] + [[ "$output" == *"3 of 3 affected pull requests"* ]] +} + +@test "warns and fails when a re-run cannot be started" { + export RERUN_FAILS=1 + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Could not re-run the milestone check for PR #100"* ]] +} + +@test "fails when the PR listing API call fails" { + mock_pr_list_failure + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list open pull requests"* ]] +} diff --git a/.github/skills/agentic-workflows/SKILL.md b/.github/skills/agentic-workflows/SKILL.md new file mode 100644 index 0000000000..ee714d339d --- /dev/null +++ b/.github/skills/agentic-workflows/SKILL.md @@ -0,0 +1,80 @@ +--- +name: agentic-workflows +description: Route gh-aw workflow design/create/debug/upgrade requests to the right prompts. +--- + +# Agentic Workflows Router + +Use this skill when a user asks to design, create, update, debug, or upgrade GitHub Agentic Workflows in this repository. + +This skill is a dispatcher: identify the task type, load the matching workflow prompt/skill file, and follow it directly. Keep responses concise and ask a clarifying question if the correct prompt is unclear. + +Read only the files you need: +Load these files from `github/gh-aw` (they are not available locally). +- `.github/aw/agentic-chat.md` +- `.github/aw/agentic-workflows-mcp.md` +- `.github/aw/asciicharts.md` +- `.github/aw/campaign.md` +- `.github/aw/charts-trending.md` +- `.github/aw/charts.md` +- `.github/aw/cli-commands.md` +- `.github/aw/context.md` +- `.github/aw/create-agentic-workflow.md` +- `.github/aw/create-shared-agentic-workflow.md` +- `.github/aw/debug-agentic-workflow.md` +- `.github/aw/dependabot.md` +- `.github/aw/deployment-status.md` +- `.github/aw/experiments.md` +- `.github/aw/github-agentic-workflows.md` +- `.github/aw/github-mcp-server.md` +- `.github/aw/llms.md` +- `.github/aw/mcp-clis.md` +- `.github/aw/memory.md` +- `.github/aw/messages.md` +- `.github/aw/network.md` +- `.github/aw/optimize-agentic-workflow.md` +- `.github/aw/patterns.md` +- `.github/aw/pr-reviewer.md` +- `.github/aw/report.md` +- `.github/aw/reuse.md` +- `.github/aw/safe-outputs-automation.md` +- `.github/aw/safe-outputs-content.md` +- `.github/aw/safe-outputs-management.md` +- `.github/aw/safe-outputs-runtime.md` +- `.github/aw/safe-outputs.md` +- `.github/aw/serena-tool.md` +- `.github/aw/shared-safe-jobs.md` +- `.github/aw/skills.md` +- `.github/aw/subagents.md` +- `.github/aw/syntax-agentic.md` +- `.github/aw/syntax-core.md` +- `.github/aw/syntax-tools-imports.md` +- `.github/aw/syntax.md` +- `.github/aw/test-coverage.md` +- `.github/aw/test-expression.md` +- `.github/aw/token-optimization.md` +- `.github/aw/triggers.md` +- `.github/aw/update-agentic-workflow.md` +- `.github/aw/upgrade-agentic-workflows.md` +- `.github/aw/visual-regression.md` +- `.github/aw/workflow-constraints.md` +- `.github/aw/workflow-editing.md` +- `.github/aw/workflow-patterns.md` + +- `.github/skills/agentic-workflow-designer/SKILL.md` +After loading the matching workflow prompt or skill, follow it directly: +- Design workflows from scratch via interview: `skills/agentic-workflow-designer/SKILL.md` +- Create new workflows: `.github/aw/create-agentic-workflow.md` +- Update existing workflows: `.github/aw/update-agentic-workflow.md` +- Debug, audit, or investigate workflows: `.github/aw/debug-agentic-workflow.md` +- Upgrade workflows and fix deprecations: `.github/aw/upgrade-agentic-workflows.md` +- Create shared components or MCP wrappers: `.github/aw/create-shared-agentic-workflow.md` +- Create report-generating workflows: `.github/aw/report.md` +- Fix Dependabot manifest PRs: `.github/aw/dependabot.md` +- Analyze coverage workflows: `.github/aw/test-coverage.md` +- Render compact markdown charts: `.github/aw/asciicharts.md` +- Map CLI commands to MCP usage: `.github/aw/cli-commands.md` +- Choose workflow architecture and patterns: `.github/aw/patterns.md` +- Optimize token usage and cost: `.github/aw/token-optimization.md` + +When the task involves OTEL, OTLP, traces, observability backends, or telemetry-driven analysis, also read and follow `skills/otel-queries/SKILL.md` after loading the matching workflow prompt or skill. diff --git a/.github/skills/generate-mstest-filter/SKILL.md b/.github/skills/generate-mstest-filter/SKILL.md index 8441823347..3d2039259b 100644 --- a/.github/skills/generate-mstest-filter/SKILL.md +++ b/.github/skills/generate-mstest-filter/SKILL.md @@ -187,7 +187,7 @@ dotnet test --list-tests --filter "" --framework < ```bash # Generate filter for "ChannelDbConnectionPoolTest" class -dotnet test tests/UnitTests/UnitTests.csproj --list-tests --filter "FullyQualifiedName~ChannelDbConnectionPoolTest" --framework net9.0 +dotnet test src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj --list-tests --filter "FullyQualifiedName~ChannelDbConnectionPoolTest" --framework net9.0 # Expected output shows matching tests: # The following Tests are available: diff --git a/.github/workflows/auto-assign-pr.yml b/.github/workflows/auto-assign-pr.yml new file mode 100644 index 0000000000..aab463f2e2 --- /dev/null +++ b/.github/workflows/auto-assign-pr.yml @@ -0,0 +1,38 @@ +name: Auto Assign PR Load Balancer + +on: + pull_request_target: + types: [opened, reopened, ready_for_review, milestoned] + +concurrency: + group: auto-assign-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + load-balance-assignees: + # Only run for assignment-eligible PRs in the upstream repository. Use + # pull_request_target so the workflow only runs once per PR event. + # Exclude PRs from forks as GitHub Actions does not have permissions to assign users on PRs from forks. + if: github.repository == 'dotnet/SqlClient' && github.event.pull_request.state == 'open' && github.event.pull_request.draft == false && github.event.pull_request.milestone != null && github.event.pull_request.head.repo.fork == false + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + pull-requests: write + env: + PR_REVIEWER_POOL: ${{ vars.PR_REVIEWER_POOL }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + sparse-checkout: | + .github/scripts/auto-assign-pr.js + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Calculate Workload and Apply + uses: actions/github-script@v9 + with: + script: | + const script = require('./.github/scripts/auto-assign-pr.js'); + await script({ github, context, core }); diff --git a/.github/workflows/check-milestone.yml b/.github/workflows/check-milestone.yml new file mode 100644 index 0000000000..8d64ae5f13 --- /dev/null +++ b/.github/workflows/check-milestone.yml @@ -0,0 +1,73 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Check Milestone +# +# Validates that every pull request has an open milestone assigned, and that +# the milestone is consistent with the branch the PR targets. +# +# Milestones map to release branches by major.minor: +# +# * "7.0.3" -> release/7.0 exists -> the PR must target release/7.0. +# * "7.1.0" -> release/7.1 does not exist and it is the earliest configured +# unbranched series -> the PR must target main. +# * "8.0.0-preview1" -> release/8.0 does not exist, but 7.1 is still the +# active unbranched series -> the PR cannot target main yet. +# +# See .github/scripts/check-milestone-branch.sh for the full rule set. +# +# Cutting release/X.Y flips the expected target for X.Y.* milestones but emits +# no pull request activity. recheck-milestones.yml reconciles the already open +# PRs that the new branch invalidates. +# +################################################################################# + +name: Check Milestone + +on: + pull_request: + # The 'edited' type covers base branch changes, so retargeting a PR (manually, + # or automatically when a stacked PR's parent merges) re-runs this check. + types: [opened, reopened, edited, synchronize, milestoned, demilestoned] + +jobs: + check-milestone: + name: Validate milestone + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + steps: + - name: Check milestone is set + if: github.event.pull_request.milestone == null + run: | + echo "::error::This PR does not have a milestone set. Please assign a milestone before merging." + exit 1 + + - name: Check milestone is open + if: github.event.pull_request.milestone != null && github.event.pull_request.milestone.state != 'open' + run: | + echo "::error::Milestone '${{ github.event.pull_request.milestone.title }}' is ${{ github.event.pull_request.milestone.state }}. Please assign an open milestone." + exit 1 + + - name: Checkout scripts + if: github.event.pull_request.milestone != null + uses: actions/checkout@v6 + with: + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Check milestone matches target branch + if: github.event.pull_request.milestone != null + env: + # Pass PR data via env to avoid script injection from milestone text. + MILESTONE_TITLE: ${{ github.event.pull_request.milestone.title }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/check-milestone-branch.sh" diff --git a/.github/workflows/cherry-pick-hotfix.yml b/.github/workflows/cherry-pick-hotfix.yml new file mode 100644 index 0000000000..678a6b20b3 --- /dev/null +++ b/.github/workflows/cherry-pick-hotfix.yml @@ -0,0 +1,113 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Cherry-pick Hotfix to Release Branch +# +# Automatically cherry-picks merged PRs into release branches when a +# "Hotfix " label is present. Supports multiple hotfix labels on +# a single PR — each one produces an independent cherry-pick PR targeting +# the corresponding release/ branch. +# +# Usage: +# 1. Merge a PR to the default branch. +# 2. Add a "Hotfix " label (e.g. "Hotfix 7.0.1") either before +# or after merging. +# 3. The workflow derives the release branch from the label's major.minor +# version (e.g. "Hotfix 7.0.1" → release/7.0) and creates a cherry-pick +# PR prefixed with "[ Cherry-pick]". +# 4. If the cherry-pick has conflicts, a placeholder PR is opened with +# "[ Cherry-pick - CONFLICTS]" and manual resolution steps. +# +################################################################################# + +name: Cherry-pick Hotfix to release branch + +# Triggers: +# - 'closed': fires at merge time — if a "Hotfix " label is already present, +# the cherry-pick runs immediately. +# - 'labeled': fires when a label is added after merge — allows retroactive cherry-picks +# by adding the label to an already-merged PR. +on: + pull_request_target: + types: [closed, labeled] + +# 'contents: write' is needed to push the cherry-pick branch. +# 'pull-requests: write' is needed to create the new PR via the GitHub CLI. +permissions: + contents: write + pull-requests: write + +jobs: + # First job: extract all hotfix versions from the PR labels and emit them as + # a JSON array so the matrix strategy can fan out one job per version. + detect-versions: + runs-on: ubuntu-latest + # Only fire for merged PRs targeting the default branch that have at least + # one "Hotfix *" label. The default-branch guard prevents recursive + # cherry-picks when a cherry-pick PR is merged into a release branch. + if: >- + github.event.pull_request.merged == true && + github.event.pull_request.base.ref == github.event.repository.default_branch && + join(github.event.pull_request.labels.*.name, ' ') != '' && + contains(join(github.event.pull_request.labels.*.name, ','), 'Hotfix ') + outputs: + versions: ${{ steps.extract.outputs.versions }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Extract hotfix versions from labels + id: extract + env: + # Pass label names via env to avoid script injection from label text. + LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + # For the 'labeled' event, this is the single label that was just added. + # For the 'closed' event this is empty, meaning all labels are processed. + EVENT_LABEL: ${{ github.event.label.name || '' }} + EVENT_ACTION: ${{ github.event.action }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/extract-hotfix-versions.sh" + + # Second job: runs once per detected version, cherry-picking the merge commit + # into each target release branch. + cherry-pick: + needs: detect-versions + if: needs.detect-versions.outputs.versions != '[]' + runs-on: ubuntu-latest + strategy: + # Don't cancel other cherry-picks if one version fails. + fail-fast: false + matrix: + version: ${{ fromJson(needs.detect-versions.outputs.versions) }} + name: Cherry-pick to release branch (${{ matrix.version }}) + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + # Full history is required so the merge commit and target branch are available + # for the cherry-pick operation. + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Cherry-pick and create PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGE_COMMIT_SHA: ${{ github.event.pull_request.merge_commit_sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_TITLE: ${{ github.event.pull_request.title }} + VERSION: ${{ matrix.version }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/cherry-pick-to-release.sh" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 64584746df..dc237459ee 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,6 +23,7 @@ on: - main - feat/** - dev/** + - release/** # Scan weekly on Saturdays at 23:33 UTC schedule: @@ -65,12 +66,16 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v5.0.1 + uses: actions/setup-dotnet@v5.2.0 with: global-json-file: global.json + + - name: Restore dotnet tools + shell: bash + run: dotnet tool restore # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL @@ -94,7 +99,7 @@ jobs: - name: Run manual build steps if: matrix.build-mode == 'manual' shell: bash - run: dotnet build src/Microsoft.Data.SqlClient.sln + run: dotnet build build.proj -t:BuildAll - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 0000000000..90ca026d37 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,26 @@ +name: "Copilot Setup Steps" + +# This workflow configures the environment for GitHub Copilot Agent with gh-aw MCP server +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + # The job MUST be called 'copilot-setup-steps' to be recognized by GitHub Copilot Agent + copilot-setup-steps: + runs-on: ubuntu-latest + + # Set minimal permissions for setup steps + # Copilot Agent receives its own token with appropriate permissions + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install gh-aw extension + uses: github/gh-aw-actions/setup-cli@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + with: + version: v0.80.9 diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml new file mode 100644 index 0000000000..d1ef3b2e67 --- /dev/null +++ b/.github/workflows/issue-triage.lock.yml @@ -0,0 +1,1851 @@ +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8b81a4840372af279438b0250d6ae7168fa2e0a2f82ae8cd52d714f18a83d425","body_hash":"99b21e9ba167d3bfbcfaeb3b04c515fc42b56679f0ee175c119bc5843f034d31","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} +# This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# +# Secrets used: +# - COPILOT_GITHUB_TOKEN +# - GH_AW_DEFAULT_OTLP_HEADERS +# - GH_AW_GITHUB_MCP_SERVER_TOKEN +# - GH_AW_GITHUB_TOKEN +# - GITHUB_TOKEN +# +# Custom actions used: +# - actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 +# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 +# +# Container images used: +# - ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 +# - ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f +# - ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e +# - ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e +# - ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699 + +name: "SqlClient Issue Auto-Triage" +on: + issue_comment: + types: + - created + issues: + types: + - opened +# roles: all # Roles processed as role check in pre-activation job + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}" + queue: max + +run-name: "SqlClient Issue Auto-Triage" + +env: + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }} + OTEL_SERVICE_NAME: gh-aw.issue-triage + OTEL_RESOURCE_ATTRIBUTES: 'gh-aw.workflow.name=SqlClient%20Issue%20Auto-Triage,gh-aw.repository=${{ github.repository }},gh-aw.run.id=${{ github.run_id }},github.run_id=${{ github.run_id }},gh-aw.engine.id=copilot' + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.GH_AW_DEFAULT_OTLP_HEADERS }} + GH_AW_OTLP_ENDPOINTS: '[{"url":"${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }}","headers":"${{ secrets.GH_AW_DEFAULT_OTLP_HEADERS }}"}]' + GH_AW_OTLP_IF_MISSING: ignore + +jobs: + activation: + if: > + github.event_name == 'issues' || + (github.event_name == 'issue_comment' + && github.event.issue.pull_request == null + && !endsWith(github.event.comment.user.login, '[bot]') + && ( + ((github.event.comment.body == '/triage' || startsWith(github.event.comment.body, '/triage ')) + && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) + || + (github.event.comment.body != '/triage' + && !startsWith(github.event.comment.body, '/triage ') + && github.event.comment.user.login == github.event.issue.user.login + && contains(github.event.issue.labels.*.name, 'Auto-Triage: Waiting for Author')) + )) + runs-on: ubuntu-slim + permissions: + actions: read + contents: read + env: + GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }} + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + body: ${{ steps.sanitized.outputs.body }} + comment_id: "" + comment_repo: "" + daily_ai_credits_exceeded: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }} + daily_ai_credits_guardrail_status: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_guardrail_status || '' }} + daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }} + daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }} + engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} + model: ${{ steps.generate_aw_info.outputs.model }} + oauth_token_check_failed: ${{ steps.check-oauth-tokens.outputs.oauth_token_check_failed == 'true' }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }} + text: ${{ steps.sanitized.outputs.text }} + title: ${{ steps.sanitized.outputs.title }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + safe-output-artifact-client: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: "auto" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AGENT_VERSION: "1.0.80" + GH_AW_INFO_CLI_VERSION: "v0.88.2" + GH_AW_INFO_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.28.12" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_AGENT_RUNTIME: "" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_aw_info.cjs')); + await main(core, context); + - name: Restore daily AIC usage cache + id: restore-daily-aic-cache + if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-workflow-usage-issuetriage-${{ github.run_id }} + restore-keys: agentic-workflow-usage-issuetriage- + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + - name: Restore daily AIC usage cache (artifact fallback) + id: restore-daily-aic-cache-fallback + if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_RESTORE_DAILY_AIC_CACHE_HIT: ${{ steps.restore-daily-aic-cache.outputs.cache-hit }} + GH_AW_RESTORE_DAILY_AIC_CACHE_MATCHED_KEY: ${{ steps.restore-daily-aic-cache.outputs.cache-matched-key }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'restore_aic_usage_cache_fallback.cjs')); + await main(); + - name: Check daily workflow token guardrail + id: daily-effective-workflow-guardrail + if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_ID: "issue-triage" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_WORKFLOW_DISPATCH_AW_CONTEXT: ${{ github.event.inputs.aw_context || '' }} + GH_AW_HAS_SLASH_COMMAND: "false" + GH_AW_HAS_LABEL_COMMAND: "false" + GH_AW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'check_daily_aic_workflow_guardrail.cjs')); + await main(); + - name: Check for OAuth tokens + id: check-oauth-tokens + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_oauth_tokens.sh" + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: | + .github + .agents + .claude + .codex + .gemini + .pi + sparse-checkout-cone-mode: true + fetch-depth: 1 + - name: Save agent config folders for base branch restoration + env: + GH_AW_AGENT_FOLDERS: ".agents .github" + GH_AW_AGENT_FILES: "AGENTS.md" + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + - name: Check workflow lock file + id: check-lock-file + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_WORKFLOW_FILE: "issue-triage.lock.yml" + GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'check_workflow_timestamp_api.cjs')); + await main(); + - name: Check compile-agentic version + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_COMPILED_VERSION: "v0.88.2" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'check_version_updates.cjs')); + await main(); + - name: Compute current body text + id: sanitized + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'compute_text.cjs')); + await main(); + - name: Log runtime features + if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" + - name: Create prompt with built-in context + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_ACTIONS_DIR: ${{ runner.temp }}/gh-aw/actions + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"file\":\"pr_context_prompt.md\",\"condition_env\":\"GH_AW_INCLUDE_PR_CONTEXT\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"}]}" + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_INCLUDE_PR_CONTEXT: ${{ (github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || github.event_name == 'pull_request_review_comment' || github.event_name == 'pull_request_review' }} + GH_AW_PROMPT_CONTENT_0000: "\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: add_comment, add_labels, remove_labels, missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0002: "\n" + GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" + GH_AW_PROMPT_CONTENT_0004: "\n" + GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/issue-triage.md}}\n" + with: + script: | + const { setupGlobals } = require(process.env.GH_AW_ACTIONS_DIR + '/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(process.env.GH_AW_ACTIONS_DIR + '/create_prompt.cjs'); + await main(core); + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_ENGINE_ID: "copilot" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'interpolate_prompt.cjs')); + await main(); + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} + GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_INCLUDE_PR_CONTEXT: ${{ (github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || github.event_name == 'pull_request_review_comment' || github.event_name == 'pull_request_review' }} + GH_AW_MCP_CLI_SERVERS_LIST: "- `github` — run `github --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + + const substitutePlaceholders = require(path.join(actionsDir, 'substitute_placeholders.cjs')); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, + GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, + GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, + GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, + GH_AW_INCLUDE_PR_CONTEXT: process.env.GH_AW_INCLUDE_PR_CONTEXT, + GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + - name: Print prompt + env: + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Stage prompt files for artifact upload + run: | + mkdir -p /tmp/gh-aw/aw-prompts + cp -a "${RUNNER_TEMP}/gh-aw/aw-prompts/." /tmp/gh-aw/aw-prompts/ + - name: Upload activation artifact + if: success() || failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: activation + include-hidden-files: true + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/models.json + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw-prompts/prompt-template.txt + /tmp/gh-aw/aw-prompts/prompt-import-tree.json + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/base + /tmp/gh-aw/.github/agents + /tmp/gh-aw/.github/skills + if-no-files-found: ignore + retention-days: 1 + + agent: + needs: activation + if: needs.activation.outputs.daily_ai_credits_exceeded != 'true' + runs-on: ubuntu-latest + environment: issue-triage + permissions: + contents: read + issues: read + pull-requests: read + timeout-minutes: 60 + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_PR_HEAD_BASE_BRANCH: "" + GH_AW_PR_HEAD_BASE_PR_NUMBER: "" + GH_AW_PR_HEAD_BASE_REF: "" + GH_AW_PR_HEAD_BASE_REPO: "" + GH_AW_PR_HEAD_BASE_SHA: "" + GH_AW_PR_HEAD_REPO: "" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + GH_AW_WORKFLOW_ID_SANITIZED: issuetriage + outputs: + agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} + ai_credits_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.ai_credits_rate_limit_error || 'false' }} + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + ambient_context: ${{ steps.parse-mcp-gateway.outputs.ambient_context }} + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + http_400_response_error: ${{ steps.detect-agent-errors.outputs.http_400_response_error || 'false' }} + inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + invocation_cap_exceeded: ${{ steps.detect-agent-errors.outputs.invocation_cap_exceeded || 'false' }} + max_cache_misses_exceeded: ${{ steps.detect-agent-errors.outputs.max_cache_misses_exceeded || 'false' }} + mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + missing_model_pricing_error: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_error || 'false' }} + missing_model_pricing_model_name: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_model_name || '' }} + model: ${{ needs.activation.outputs.model }} + model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} + setup-span-id: ${{ steps.setup.outputs.span-id }} + setup-trace-id: ${{ steps.setup.outputs.trace-id }} + shell_expansion_guard_rejected: ${{ steps.detect-agent-errors.outputs.shell_expansion_guard_rejected || 'false' }} + unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Set runtime paths + id: set-runtime-paths + env: + GH_AW_RUNNER_TOOL_CACHE: ${{ runner.tool_cache }} + run: | + if [ -z "${RUNNER_TOOL_CACHE:-}" ]; then + echo "RUNNER_TOOL_CACHE=${GH_AW_RUNNER_TOOL_CACHE}" >> "$GITHUB_ENV" + fi + { + echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" + echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" + echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" + } >> "$GITHUB_OUTPUT" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" + - name: Check OTLP telemetry configuration + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_otlp_default_credentials.sh" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Create gh-aw temp directory + run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh" + - name: Configure gh CLI for GitHub Enterprise + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" + env: + GH_TOKEN: ${{ github.token }} + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request || github.event.issue.pull_request || github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.aw_context || '{}').item_type == 'pull_request' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'checkout_pr_branch.cjs')); + await main(); + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" + env: + GH_HOST: github.com + GH_AW_COMPILED_VERSION: v0.88.2 + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.12 --rootless + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + GH_AW_GITHUB_MIN_INTEGRITY: 'none' + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); + await determineAutomaticLockdown(github, context, core); + - name: Parse integrity filter lists + id: parse-guard-vars + env: + GH_AW_BLOCKED_USERS_VAR: ${{ vars.GH_AW_GITHUB_BLOCKED_USERS || '' }} + GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} + GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" + - name: Restore agent config folders from base branch + if: steps.checkout-pr.outcome == 'success' + env: + GH_AW_AGENT_FOLDERS: ".agents .github" + GH_AW_AGENT_FILES: "AGENTS.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" + - name: Restore inline sub-agents from activation artifact + env: + GH_AW_SUB_AGENT_DIR: ".github/agents" + GH_AW_SUB_AGENT_EXT: ".agent.md" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh" + - name: Restore inline skills from activation artifact + env: + GH_AW_SKILL_DIR: ".github/skills" + run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699 + - name: Prepare Safe Outputs Directories + run: | + mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + - name: Generate Safe Outputs Config + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" + GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"add_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"remove_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"report_incomplete\":{}}" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'create_files.cjs')); + await main(); + - name: Generate Safe Outputs Tools + env: + GH_AW_TOOLS_META_JSON: | + { + "description_suffixes": { + "add_comment": " CONSTRAINTS: Maximum 1 comment(s) can be added. Supports reply_to_id for discussion threading.", + "add_labels": " CONSTRAINTS: Maximum 1 label(s) can be added. Only these labels are allowed: [\"Auto-Triage: Waiting for Author\"].", + "remove_labels": " CONSTRAINTS: Maximum 1 label(s) can be removed. Only these labels can be removed: [Auto-Triage: Waiting for Author]." + }, + "repo_params": {}, + "dynamic_tools": [] + } + GH_AW_VALIDATION_JSON: | + { + "add_comment": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "comment_id": { + "optionalPositiveInteger": true + }, + "item_number": { + "issueOrPRNumber": true + }, + "pr": { + "issueOrPRNumber": true + }, + "pr_number": { + "issueOrPRNumber": true + }, + "reply_to_id": { + "type": "string", + "maxLength": 256 + }, + "repo": { + "type": "string", + "maxLength": 256 + }, + "target": { + "type": "string", + "enum": [ + "status" + ] + }, + "temporary_id": { + "type": "string", + "pattern": "^#?aw_[A-Za-z0-9_]{3,12}$" + } + } + }, + "add_labels": { + "defaultMax": 5, + "fields": { + "item_number": { + "issueNumberOrTemporaryId": true + }, + "labels": { + "required": true, + "type": "array" + }, + "repo": { + "type": "string", + "maxLength": 256 + } + } + }, + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + }, + "remove_labels": { + "defaultMax": 5, + "fields": { + "item_number": { + "issueNumberOrTemporaryId": true + }, + "labels": { + "required": true, + "type": "array" + }, + "repo": { + "type": "string", + "maxLength": 256 + } + } + }, + "report_incomplete": { + "defaultMax": 5, + "fields": { + "details": { + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 1024 + } + } + } + } + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_safe_outputs_tools.cjs')); + await main(); + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }} + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} + GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" + if [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -r "${GITHUB_EVENT_PATH}" ]; then + GH_AW_SAFEOUTPUTS_EVENT_PATH="${RUNNER_TEMP}/gh-aw/safeoutputs/github_event.json" + cp "${GITHUB_EVENT_PATH}" "${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + export GITHUB_EVENT_PATH="${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + fi + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="8080" + export MCP_GATEWAY_DOMAIN="awmg-mcpg" + export MCP_GATEWAY_HOST_DOMAIN="localhost" + MCP_GATEWAY_AGENT_ID=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_AGENT_ID}" + export MCP_GATEWAY_AGENT_ID + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export MCP_GATEWAY_ALLOWED_MOUNT_ROOTS="${GITHUB_WORKSPACE}:rw,${RUNNER_TEMP}/gh-aw:ro,${RUNNER_TEMP}/gh-aw/safeoutputs:rw,/opt:ro,/tmp:rw,/usr/bin/gh:ro" + export GH_AW_PR_HEAD_BASE_BRANCH="${GH_AW_PR_HEAD_BASE_BRANCH:-}" + export GH_AW_PR_HEAD_BASE_SHA="${GH_AW_PR_HEAD_BASE_SHA:-}" + export GH_AW_PR_HEAD_BASE_REPO="${GH_AW_PR_HEAD_BASE_REPO:-}" + export GH_AW_PR_HEAD_BASE_PR_NUMBER="${GH_AW_PR_HEAD_BASE_PR_NUMBER:-}" + export GH_AW_PR_HEAD_BASE_REF="${GH_AW_PR_HEAD_BASE_REF:-}" + export GH_AW_PR_HEAD_REPO="${GH_AW_PR_HEAD_REPO:-}" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') + MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') + source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_AGENT_ID -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_PR_HEAD_BASE_BRANCH -e GH_AW_PR_HEAD_BASE_SHA -e GH_AW_PR_HEAD_BASE_REPO -e GH_AW_PR_HEAD_BASE_PR_NUMBER -e GH_AW_PR_HEAD_BASE_REF -e GH_AW_PR_HEAD_REPO -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e RUNNER_TOOL_CACHE -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.15' + + mkdir -p "$HOME/.copilot" + GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) + cat << GH_AW_MCP_CONFIG_137b94c134aafdc9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v1.11.0", + "env": { + "GITHUB_FEATURES": "fields_param", + "GITHUB_HOST": "${GITHUB_SERVER_URL}", + "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" + }, + "guard-policies": { + "allow-only": { + "approval-labels": ${{ steps.parse-guard-vars.outputs.approval_labels }}, + "blocked-users": ${{ steps.parse-guard-vars.outputs.blocked_users }}, + "min-integrity": "none", + "repos": "all", + "trusted-users": ${{ steps.parse-guard-vars.outputs.trusted_users }} + } + } + }, + "safeoutputs": { + "type": "stdio", + "container": "ghcr.io/github/gh-aw-node", + "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "${RUNNER_TEMP}/gh-aw/safeoutputs:${RUNNER_TEMP}/gh-aw/safeoutputs:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"], + "args": ["-w", "\${GITHUB_WORKSPACE}"], + "entrypoint": "sh", + "entrypointArgs": ["-c", "sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh"], + "env": { + "DEBUG": "*", + "DEFAULT_BRANCH": "\${DEFAULT_BRANCH}", + "GH_AW_ASSETS_ALLOWED_EXTS": "\${GH_AW_ASSETS_ALLOWED_EXTS}", + "GH_AW_ASSETS_BRANCH": "\${GH_AW_ASSETS_BRANCH}", + "GH_AW_ASSETS_MAX_SIZE_KB": "\${GH_AW_ASSETS_MAX_SIZE_KB}", + "GH_AW_MCP_LOG_DIR": "\${GH_AW_MCP_LOG_DIR}", + "GH_AW_SAFE_OUTPUTS": "\${GH_AW_SAFE_OUTPUTS}", + "GH_AW_SAFE_OUTPUTS_CONFIG_PATH": "\${GH_AW_SAFE_OUTPUTS_CONFIG_PATH}", + "GH_AW_SAFE_OUTPUTS_TOOLS_PATH": "\${GH_AW_SAFE_OUTPUTS_TOOLS_PATH}", + "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST": "\${GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST}", + "GH_AW_PR_HEAD_BASE_BRANCH": "\${GH_AW_PR_HEAD_BASE_BRANCH}", + "GH_AW_PR_HEAD_BASE_SHA": "\${GH_AW_PR_HEAD_BASE_SHA}", + "GH_AW_PR_HEAD_BASE_REPO": "\${GH_AW_PR_HEAD_BASE_REPO}", + "GH_AW_PR_HEAD_BASE_PR_NUMBER": "\${GH_AW_PR_HEAD_BASE_PR_NUMBER}", + "GH_AW_PR_HEAD_BASE_REF": "\${GH_AW_PR_HEAD_BASE_REF}", + "GH_AW_PR_HEAD_REPO": "\${GH_AW_PR_HEAD_REPO}", + "GITHUB_EVENT_NAME": "\${GITHUB_EVENT_NAME}", + "GITHUB_EVENT_PATH": "\${GITHUB_EVENT_PATH}", + "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}", + "GITHUB_SHA": "\${GITHUB_SHA}", + "GITHUB_TOKEN": "\${GITHUB_TOKEN}", + "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}", + "RUNNER_TEMP": "\${RUNNER_TEMP}" + }, + "guard-policies": { + "write-sink": { + "accept": [ + "*" + ], + "sink-visibility": "${GH_AW_SINK_VISIBILITY}" + } + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "agentId": "${MCP_GATEWAY_AGENT_ID}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}", + "startupTimeout": 120, + "opentelemetry": { + "endpoint": "${OTEL_EXPORTER_OTLP_ENDPOINT}", + "traceId": "${GITHUB_AW_OTEL_TRACE_ID}", + "spanId": "${GITHUB_AW_OTEL_PARENT_SPAN_ID}" + } + } + } + GH_AW_MCP_CONFIG_137b94c134aafdc9_EOF + - name: Mount MCP servers as CLIs + id: mount-mcp-clis + continue-on-error: true + env: + MCP_GATEWAY_AGENT_ID: ${{ steps.start-mcp-gateway.outputs.gateway-agent-id }} + MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io); + const { main } = require(path.join(actionsDir, 'mount_mcp_as_cli.cjs')); + await main(); + - name: Clean credentials + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh" + - name: Audit pre-agent workspace + id: pre_agent_audit + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh" + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool github + # --allow-tool safeoutputs + # --allow-tool shell(cat) + # --allow-tool shell(date) + # --allow-tool shell(echo) + # --allow-tool shell(find) + # --allow-tool shell(github:*) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(ls) + # --allow-tool shell(printf) + # --allow-tool shell(pwd) + # --allow-tool shell(safeoutputs:*) + # --allow-tool shell(sort) + # --allow-tool shell(tail) + # --allow-tool shell(uniq) + # --allow-tool shell(wc) + # --allow-tool shell(yq) + # --allow-tool write + timeout-minutes: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"; if [ "$gh_aw_exit_code" -ne 0 ]; then echo "::error::Agent execution exited with code $gh_aw_exit_code"; fi' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + export GH_AW_MCP_CONFIG="$HOME/.copilot/mcp-config.json" + GH_AW_COPILOT_SRC="$(command -v copilot 2>/dev/null || true)" + if [ -z "$GH_AW_COPILOT_SRC" ] || [ ! -x "$GH_AW_COPILOT_SRC" ]; then + echo "GitHub Copilot CLI executable not found on PATH after installation" >&2 + exit 127 + fi + GH_AW_COPILOT_BIN="${RUNNER_TEMP}/gh-aw/bin/copilot" + mkdir -p "${RUNNER_TEMP}/gh-aw/bin" + if [ "$GH_AW_COPILOT_SRC" != "$GH_AW_COPILOT_BIN" ]; then + cp "$GH_AW_COPILOT_SRC" "$GH_AW_COPILOT_BIN" + fi + chmod 755 "$GH_AW_COPILOT_BIN" + + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/agent-stdio.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-1000}" + if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then + GH_AW_MAX_AI_CREDITS="1000" + fi + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.12/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"],\"isolation\":true,\"topologyAttach\":[\"awmg-mcpg\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.12,squid=sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f,agent=sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202,api-proxy=sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32,cli-proxy=sha256:5250629d48eaedfedf2e948785228e8da29eec2a83cbab58ea0751c14a7b021d\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + GH_AW_AWF_ENGINE_NAME=copilot \ + GH_AW_AWF_HARNESS_MARKER='[copilot-harness]' \ + GH_AW_AWF_LOG_FILE=/tmp/gh-aw/agent-stdio.log \ + GH_AW_AWF_ATTEMPT_LOG_NAME=copilot \ + bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: auto + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: agent + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_TIMEOUT_MINUTES: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + GH_AW_VERSION: v0.88.2 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Detect agent errors + if: always() + id: detect-agent-errors + continue-on-error: true + env: + GH_AW_AGENTIC_EXECUTION_OUTCOME: ${{ steps.agentic_execution.outcome }} + GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'detect_agent_errors.cjs')); + await main(); + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/copy_copilot_session_state.sh" + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_AGENT_ID: ${{ steps.start-mcp-gateway.outputs.gateway-agent-id }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'redact_secrets.cjs')); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Append agent step summary + if: always() + run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh" + - name: Copy Safe Outputs + if: always() + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + run: | + mkdir -p /tmp/gh-aw + cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'collect_ndjson_output.cjs')); + await main(); + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_copilot_log.cjs')); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_mcp_gateway_log.cjs')); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_firewall_logs.sh" --rootless + - name: Parse token usage for step summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); + await main(); + - name: Print AWF reflect summary + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'awf_reflect_summary.cjs')); + await main(); + - name: Generate observability summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_observability_summary.cjs')); + await main(core); + - name: Write agent output placeholder if missing + if: always() + run: | + if [ ! -f /tmp/gh-aw/agent_output.json ]; then + echo '{"items":[]}' > /tmp/gh-aw/agent_output.json + fi + # Small dedicated copy of the agent output so safe-output processing + # survives a failed or timed-out upload of the larger agent artifact + - name: Upload agent output fallback artifact + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-output-fallback + path: | + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/safeoutputs.jsonl + if-no-files-found: ignore + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/proxy-logs/ + !/tmp/gh-aw/proxy-logs/proxy-tls/ + /tmp/gh-aw/agent_usage.json + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/pre-agent-audit.txt + /tmp/gh-aw/agent/ + /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/otel.jsonl + /tmp/gh-aw/otlp-export-errors.jsonl + /tmp/gh-aw/safeoutputs.jsonl + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/aw-*.patch + /tmp/gh-aw/aw-*.bundle + /tmp/gh-aw/awf-config.json + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/sandbox/firewall/audit/ + /tmp/gh-aw/sandbox/firewall/awf-reflect.json + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - safe_outputs + if: > + always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || + needs.activation.outputs.oauth_token_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true' || + needs.activation.outputs.daily_ai_credits_exceeded == 'true') + runs-on: ubuntu-slim + environment: issue-triage + permissions: + actions: read + issues: write + pull-requests: write + concurrency: + group: "gh-aw-conclusion-issue-triage" + cancel-in-progress: false + queue: max + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Download detection artifact + id: download-detection-artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: detection + path: /tmp/gh-aw/threat-detection/ + - name: Download Safe Outputs Items Manifest + id: download-safe-outputs-manifest + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: safe-outputs-items + merge-multiple: true + path: /tmp/gh-aw/ + - name: Collect usage artifact files + if: always() + continue-on-error: true + run: bash "${RUNNER_TEMP}/gh-aw/actions/collect_usage_artifact_files.sh" + - name: Upload usage artifact + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: usage + path: | + /tmp/gh-aw/usage/aw_info.json + /tmp/gh-aw/usage/aw-info.jsonl + /tmp/gh-aw/usage/agent_usage.json + /tmp/gh-aw/usage/agent_usage.jsonl + /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/evals.jsonl + /tmp/gh-aw/usage/graders/grader_manifest.json + /tmp/gh-aw/usage/graders/grader_results.json + /tmp/gh-aw/usage/github_rate_limits.jsonl + /tmp/gh-aw/usage/agent/token_usage.jsonl + /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json + if-no-files-found: ignore + - name: Restore daily AIC usage cache + id: restore-daily-aic-cache-conclusion + if: always() + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-workflow-usage-issuetriage-${{ github.run_id }} + restore-keys: agentic-workflow-usage-issuetriage- + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + - name: Write daily AIC usage cache entry + id: write-daily-aic-cache + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context); + const { main } = require(path.join(actionsDir, 'write_daily_aic_usage_cache.cjs')); + await main(); + - name: Save daily AIC usage cache + id: save-daily-aic-cache + if: always() + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-workflow-usage-issuetriage-${{ github.run_id }} + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + - name: Upload daily AIC usage cache artifact + id: upload-daily-aic-cache + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: aic-usage-cache + path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl + if-no-files-found: ignore + retention-days: 7 + - name: Process no-op messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_REPORT_AS_ISSUE: "false" + GH_AW_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} + GH_AW_WORKFLOW_ID: "issue-triage" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_noop_message.cjs')); + await main(); + - name: Log detection run + id: detection_runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_detection_runs.cjs')); + await main(); + - name: Record missing tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'missing_tool.cjs')); + await main(); + - name: Record incomplete + id: report_incomplete + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'report_incomplete_handler.cjs')); + await main(); + - name: Handle agent failure + id: handle_agent_failure + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "issue-triage" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "0" + GH_AW_ENGINE_ID: "copilot" + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} + GH_AW_AI_CREDITS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.ai_credits_rate_limit_error || 'false' }} + GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} + GH_AW_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} + GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} + GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_HTTP_400_RESPONSE_ERROR: ${{ needs.agent.outputs.http_400_response_error }} + GH_AW_MAX_CACHE_MISSES_EXCEEDED: ${{ needs.agent.outputs.max_cache_misses_exceeded }} + GH_AW_MISSING_MODEL_PRICING_ERROR: ${{ needs.agent.outputs.missing_model_pricing_error }} + GH_AW_MISSING_MODEL_PRICING_MODEL_NAME: ${{ needs.agent.outputs.missing_model_pricing_model_name }} + GH_AW_SHELL_EXPANSION_GUARD_REJECTED: ${{ needs.agent.outputs.shell_expansion_guard_rejected }} + GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" + GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} + GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} + GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} + GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} + GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" + GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" + GH_AW_TIMEOUT_MINUTES: "${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_agent_failure.cjs')); + await main(); + - name: Report failed jobs + id: report_failed_jobs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_REPORT_FAILED_JOBS: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'report_failed_jobs.cjs')); + await main(); + + detection: + needs: + - activation + - agent + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + environment: issue-triage + permissions: + contents: read + timeout-minutes: 10 + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse_detection_token_usage.outputs.aic }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_reason: ${{ steps.detection_conclusion.outputs.reason }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download activation artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Checkout repository for patch context + if: needs.agent.outputs.has_patch == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # --- Threat Detection --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f + - name: Check if detection needed + id: detection_guard + if: always() + env: + OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP Config for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json" + rm -f "$HOME/.copilot/mcp-config.json" + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/prepare_threat_detection_files.sh" + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + WORKFLOW_DESCRIPTION: "No description provided" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'setup_threat_detection.cjs')); + await main(); + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.12 --rootless + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" + env: + GH_HOST: github.com + GH_AW_COMPILED_VERSION: v0.88.2 + - name: Install threat-detect binary + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/install_threat_detect_binary.sh" v0.5.1 + - name: Execute threat detection with AWF + id: detection_agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + timeout-minutes: 10 + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: auto + GH_AW_HARNESS_MAX_RETRIES: 0 + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: detection + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 10 + GH_AW_VERSION: v0.88.2 + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + WORKFLOW_DESCRIPTION: "No description provided" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + GH_AW_COPILOT_SRC="$(command -v copilot 2>/dev/null || true)" + if [ -z "$GH_AW_COPILOT_SRC" ] || [ ! -x "$GH_AW_COPILOT_SRC" ]; then + echo "GitHub Copilot CLI executable not found on PATH after installation" >&2 + exit 127 + fi + GH_AW_COPILOT_BIN="${RUNNER_TEMP}/gh-aw/bin/copilot" + mkdir -p "${RUNNER_TEMP}/gh-aw/bin" + if [ "$GH_AW_COPILOT_SRC" != "$GH_AW_COPILOT_BIN" ]; then + cp "$GH_AW_COPILOT_SRC" "$GH_AW_COPILOT_BIN" + fi + chmod 755 "$GH_AW_COPILOT_BIN" + + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then + GH_AW_MAX_AI_CREDITS="400" + fi + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.12/awf-config.schema.json\",\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.12,squid=sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f,agent=sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202,api-proxy=sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32,cli-proxy=sha256:5250629d48eaedfedf2e948785228e8da29eec2a83cbab58ea0751c14a7b021d\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + _GH_AW_CHROOT_JSON=$(jq -c --arg src "${RUNNER_TEMP}/gh-aw" --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home "${RUNNER_TEMP}/gh-aw/home" '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; } + printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --mount /tmp/gh-aw:/tmp/gh-aw:rw --mount /tmp/gh-aw/threat-detection:/tmp/gh-aw/threat-detection:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && threat-detect --engine copilot --output /tmp/gh-aw/threat-detection/detection_result.json /tmp/gh-aw/threat-detection' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + - name: Render detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'render_detection_log.cjs')); + await main(); + - name: Copy detection firewall logs + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + run: | + mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall + if [ -d /tmp/gh-aw/sandbox/firewall/logs ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/logs && cp -r /tmp/gh-aw/sandbox/firewall/logs/. /tmp/gh-aw/threat-detection/sandbox/firewall/logs/; fi + if [ -d /tmp/gh-aw/sandbox/firewall/audit ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/audit && cp -r /tmp/gh-aw/sandbox/firewall/audit/. /tmp/gh-aw/threat-detection/sandbox/firewall/audit/; fi + - name: Upload threat detection artifact + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: detection + path: | + /tmp/gh-aw/threat-detection/detection_result.json + /tmp/gh-aw/threat-detection/sandbox/firewall/logs/ + /tmp/gh-aw/threat-detection/sandbox/firewall/audit/ + if-no-files-found: ignore + - name: Parse threat detection token usage for step summary + id: parse_detection_token_usage + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); + await main(); + - name: Conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json + + safe_outputs: + needs: + - activation + - agent + - detection + if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success' + runs-on: ubuntu-slim + environment: issue-triage + permissions: + issues: write + pull-requests: write + timeout-minutes: 45 + env: + GH_AW_AGENT_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_AIC: ${{ needs.agent.outputs.aic }} + GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/issue-triage" + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} + GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} + GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} + GH_AW_ENGINE_ID: "copilot" + GH_AW_ENGINE_MODEL: "auto" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_WORKFLOW_ID: "issue-triage" + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + comment_id: ${{ steps.process_safe_outputs.outputs.comment_id }} + comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_items_applied: ${{ steps.process_safe_outputs.outputs.items_applied }} + process_safe_outputs_items_cancelled: ${{ steps.process_safe_outputs.outputs.items_cancelled }} + process_safe_outputs_items_deferred: ${{ steps.process_safe_outputs.outputs.items_deferred }} + process_safe_outputs_items_failed: ${{ steps.process_safe_outputs.outputs.items_failed }} + process_safe_outputs_items_skipped: ${{ steps.process_safe_outputs.outputs.items_skipped }} + process_safe_outputs_items_succeeded: ${{ steps.process_safe_outputs.outputs.items_succeeded }} + process_safe_outputs_items_warnings: ${{ steps.process_safe_outputs.outputs.items_warnings }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_status: ${{ steps.process_safe_outputs.outputs.status }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + id: setup + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: "{agent,agent-output-fallback}" + merge-multiple: true + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Configure GH_HOST for enterprise compatibility + id: ghes-host-config + shell: bash + run: | # zizmor: ignore[github-env] - GITHUB_SERVER_URL is set by GitHub Actions, not user input. + # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct + # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op. + GH_HOST="${GITHUB_SERVER_URL#https://}" + GH_HOST="${GH_HOST#http://}" + echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"add_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"remove_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"report_incomplete\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'process_safe_outputs.cjs')); + await main(); + - name: Upload Safe Outputs Items + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-outputs-items + path: | + /tmp/gh-aw/safe-output-items.jsonl + /tmp/gh-aw/temporary-id-map.json + /tmp/gh-aw/safe-output-errors.json + if-no-files-found: ignore diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md new file mode 100644 index 0000000000..100d8baf54 --- /dev/null +++ b/.github/workflows/issue-triage.md @@ -0,0 +1,286 @@ +--- +on: + issues: + types: [opened] + issue_comment: + types: [created] + roles: all + +# Cheap gate evaluated BEFORE the agent boots. The activation job is skipped +# (zero compute, $0) if this is false. Only events matching one of the +# following three conditions cause the workflow to run: +# +# 1. issues.opened +# -> Initial triage. Always runs. +# +# 2. issue_comment.created from the issue's original author, on an issue +# (not a PR), not a bot, AND the issue currently has the label +# "Auto-Triage: Waiting for Author". +# -> Follow-up triage. The label is applied by the initial triage +# only when environment fields were missing, and removed by the +# follow-up triage once the author supplies them. Without the +# label, author comments do NOT boot the agent. +# +# 3. issue_comment.created whose body starts with "/triage", from a repo +# OWNER, MEMBER, or COLLABORATOR (maintainer-only on-demand override). +# -> On-demand triage. Bypasses the follow-up gate; produces a fresh +# triage summary regardless of label or prior summaries. +if: | + github.event_name == 'issues' || + (github.event_name == 'issue_comment' + && github.event.issue.pull_request == null + && !endsWith(github.event.comment.user.login, '[bot]') + && ( + ((github.event.comment.body == '/triage' || startsWith(github.event.comment.body, '/triage ')) + && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) + || + (github.event.comment.body != '/triage' + && !startsWith(github.event.comment.body, '/triage ') + && github.event.comment.user.login == github.event.issue.user.login + && contains(github.event.issue.labels.*.name, 'Auto-Triage: Waiting for Author')) + )) + +engine: copilot +model: auto + +environment: issue-triage + +permissions: + contents: read + issues: read + pull-requests: read + +tools: + bash: [cat, find, grep] + github: + min-integrity: none + +safe-outputs: + # One triage summary per run. `hide-older-comments` collapses previous + # summaries so only the latest is visible. + add-comment: + max: 1 + hide-older-comments: true + # Allow the workflow to apply/remove ONLY this one internal-state label. + # The label is the YAML-level flag that lets the cheap `if:` gate above + # decide whether an author comment should boot the agent at all. + add-labels: + allowed: ["Auto-Triage: Waiting for Author"] + max: 1 + remove-labels: + allowed: ["Auto-Triage: Waiting for Author"] + max: 1 + # Silently skip noop runs (e.g. random author comment with no new env info). + # Without this, gh-aw auto-creates a tracking issue "[aw] No-Op Runs" and + # appends a comment to it for every noop. + noop: + report-as-issue: false +--- + +# SqlClient Issue Auto-Triage + +You are a triage specialist for **Microsoft.Data.SqlClient**. +Your job is to post **at most one** triage summary comment per workflow run +using `add_comment`. + +This workflow runs in three situations. Identify which one **before** doing +any work, then follow the matching flow: + +1. **Initial triage** — `event_name == "issues"`. A new issue was just opened. + Always proceed to the triage instructions below. +2. **Follow-up triage** — `event_name == "issue_comment"` and the comment body + does NOT start with `/triage`. The workflow-level `if:` has already verified + the issue currently carries the label `Auto-Triage: Waiting for Author`, + so a prior triage flagged missing env info and the author has now responded. + Treat later author comments as part of the issue body and re-validate the + environment. There are three sub-cases routed by "Follow-up routing" under + Instructions: + - **No progress** (comment supplied no new env field, e.g. "will share + soon") → silent `noop`, label stays, no comment posted. + - **Partial progress** (comment supplied at least one new env field but + others are still missing) → post a fresh summary acknowledging what + was provided and re-asking for the rest, KEEP the label. + - **Complete** (all required env fields are now present) → post a fresh + summary, REMOVE the label. +3. **On-demand triage** — `event_name == "issue_comment"` and the comment body + starts with `/triage`. A maintainer is explicitly requesting a fresh triage. + Ignore label state and prior summary counts; proceed to the triage + instructions and produce a new summary. Do NOT change the label as part of + `/triage` runs (leave it as-is). + +Do NOT call `add_comment` more than once per run. +Do NOT call `add_labels` or `remove_labels` for any label other than +`Auto-Triage: Waiting for Author` — that single label is the only one this +workflow is permitted to manage. +Do NOT post intermediate findings. Do NOT post separate comments for +area detection, duplicate checking, or environment validation. +Everything goes into the single triage summary at the end. + +--- + +## Label-managed state + +The workflow uses one internal-state label to decide cheaply (at the YAML +`if:` level) whether an author comment should boot the agent at all: + +- **`Auto-Triage: Waiting for Author`** — present iff the most recent + triage summary flagged `⚠️ Missing:` or `⚠️ Partial:` environment fields + and we are waiting for the issue author to supply them. + +The agent (you) is responsible for keeping this label accurate — see the +"Actions" section below for exactly when to call `add_labels` / +`remove_labels`. + +Only if the workflow-level `if:` gate above evaluated to true, proceed to the +triage instructions below and produce a fresh summary. Treat the prior summary +as **invalidated** — the new one supersedes it (the older one will be collapsed +automatically by `hide-older-comments`). + +--- + +## Required Context + +Before analyzing the issue, you MUST read all project knowledge base files +from the checked-out repository. Recursively list the `.github/` directory +and read every markdown file (`.md`) found under it, excluding the `workflows/` +subdirectory. This includes but is not limited to instructions, prompts, +issue templates, skills, plans, and any other documentation files present. + +Use these files to inform your area classification, duplicate detection, +environment validation, and analysis. Do not skip this step. + +--- + +## Instructions + +Read the issue body **and, for follow-up / on-demand runs, every subsequent +comment**. Then do ALL of the following analysis silently (using read tools +and search only — no comments, no outputs): + +### Follow-up routing (scenario 2 only) + +Before running the full analysis on a follow-up run, decide which of three +sub-cases this comment falls into. Use the rules in step **B** below to +determine which environment fields each source supplies. + +Compute two snapshots: + +- **BEFORE** = env fields supplied by the issue body + every author comment + EXCEPT the triggering comment. +- **AFTER** = env fields supplied by the issue body + every author comment + INCLUDING the triggering comment. + +Then route as follows: + +1. **No progress** — `AFTER == BEFORE` (the new comment did not supply any + new env field; e.g. "okay, will share details soon", a question, an + unrelated remark). → Call `noop` with a short reason like `"Author + commented but supplied no new env info"` and STOP. Do NOT call + `add_comment`. Do NOT change the label. The label stays so the next + author comment can re-trigger this workflow. + +2. **Partial progress** — `AFTER` adds at least one new env field but is + still incomplete (some required fields are still missing). → Proceed + to the full analysis below and post a fresh triage summary. The + `Environment` row MUST acknowledge what was just provided and list + only the fields that are STILL missing, e.g. + `⚠️ Partial: received SqlClient version and OS; still missing: .NET TFM, SQL Server version`. + Keep the label `Auto-Triage: Waiting for Author` on the issue (i.e. call + `add_labels` with that label — it is a no-op if already present). + +3. **Complete** — `AFTER` contains every required env field. → Proceed to + the full analysis below and post a fresh triage summary. The + `Environment` row says `All required environment details provided for + investigation`. Call `remove_labels` with the label. + +This routing does NOT apply to initial triage (scenario 1) or on-demand +`/triage` (scenario 3) — those always produce a fresh summary using the +standard label-management rules in the Actions section below. + +### Full analysis + +**A. Classify issue type**: Bug (reports unexpected behavior, crash, regression, or incorrect results), Feature (has proposal), Question, or Task. + +**B. Validate environment** (bugs only): Check for these required fields: +SqlClient version, .NET target framework, SQL Server version, OS, +repro steps, expected vs actual behavior. +If any are missing, list them explicitly in the triage summary (e.g. "Missing: SQL Server version, OS"). +For follow-up runs, treat information supplied in any later comment by the +issue author as if it were part of the original issue body. +Proceed with all remaining triage steps regardless of missing environment details. + +**C. Classify area**: Based on the issue content, pick the single best matching area label from this list: + +| Label | Scope | +|-------|-------| +| `Area\Connection Pooling` | Pool behavior, timeouts, pool size, pool exhaustion | +| `Area\AKV Provider` | Always Encrypted Azure Key Vault provider | +| `Area\Json` | JSON data type support | +| `Area\Managed SNI` | Managed SNI / network layer | +| `Area\Native SNI` | Native SNI / network layer | +| `Area\Sql Bulk Copy` | SqlBulkCopy operations | +| `Area\Netcore` | .NET runtime / netcore specific | +| `Area\Netfx` | .NET Framework specific | +| `Area\Tests` | Test code / test projects | +| `Area\Documentation` | Docs and samples | +| `Area\Azure Connectivity` | Azure connectivity | +| `Area\Engineering` | Build, CI/CD, infrastructure | +| `Area\Vector` | Vector feature | +| `Area\Async` | Async operations | + +**D. Search for duplicates**: Search `repo:dotnet/SqlClient ` for similar issues. + +**E. Check for regression**: If the reporter mentions a previously working version, note the version boundary. + +--- + +## Actions + +Call `add_comment` exactly **once** with this markdown. For follow-up runs +add "(updated after author response)" to the heading; for on-demand `/triage` +runs add "(on-demand re-triage)" to the heading: + +``` +## 🔍 Triage Summary + +| Check | Result | +|-------|--------| +| Issue type | | +| Environment | ; still missing: / ⚠️ Missing: list specific fields> | +| Area | | +| Duplicates | | +| Regression | | + +### Analysis + +<2-4 sentences: what the issue is about, which component is likely affected, +and severity assessment (P0-P3)> + +### Next Steps + + handling logic present in the synchronous path.") +- If duplicates were found: recommend reviewing the linked issues before proceeding. +- If regression: note the version boundary and state that bisection is recommended.> + +> **Note**: This triage summary is auto-generated by an AI agent. The analysis and suggestions above have not been verified by a human maintainer. Please treat as preliminary guidance only. +``` + +**Then manage the label**: + +- For **on-demand `/triage` runs**, do NOT touch the label — preserve + whatever state existed before. +- For **initial triage** and **follow-up triage**, base the decision on the + `Environment` row of the summary you just posted: + - If it contains `⚠️ Missing:` or `⚠️ Partial:` → call `add_labels` with + `["Auto-Triage: Waiting for Author"]` (no-op if already present). + - Otherwise (env complete) → call `remove_labels` with + `["Auto-Triage: Waiting for Author"]` (safe to call even if absent). + +If the issue is spam or no action is needed, call the `noop` tool instead. \ No newline at end of file diff --git a/.github/workflows/notify-author-attention.yml b/.github/workflows/notify-author-attention.yml new file mode 100644 index 0000000000..4799e9efd1 --- /dev/null +++ b/.github/workflows/notify-author-attention.yml @@ -0,0 +1,35 @@ +name: Notify Author attention needed + +on: + pull_request_target: + types: [labeled] + +jobs: + notify-author: + # Only run when 'Author attention needed' label is added to a PR + if: >- + github.repository == 'dotnet/SqlClient' && + github.event.label.name == 'Author attention needed' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Post comment to PR author + uses: actions/github-script@v9 + with: + script: | + const issue_number = context.payload.pull_request.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + const author = context.payload.pull_request.user.login; + + const body = `@${author} This pull request has been marked as **Author attention needed**.\n\nWhen you have addressed the reviewer feedback and are ready for another review, please post a comment with \`/ready\` to remove the label and re-engage reviewers.`; + + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body, + }); + core.info(`Posted notification comment on PR #${issue_number}`); diff --git a/.github/workflows/recheck-milestones.yml b/.github/workflows/recheck-milestones.yml new file mode 100644 index 0000000000..679b76f144 --- /dev/null +++ b/.github/workflows/recheck-milestones.yml @@ -0,0 +1,65 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Recheck Milestones +# +# Reconciles open pull requests when a release branch is cut. +# +# check-milestone.yml decides where a milestone belongs by asking whether +# release/. exists and which milestone series is active on the +# default branch. Creating a release branch can invalidate X.Y.* pull requests +# and make the next series eligible, but emits no pull request activity, so +# already open pull requests would otherwise keep their previous verdicts. +# +# This lives in its own workflow so that check-milestone.yml stays purely pull +# request scoped, and so the elevated 'actions: write' permission needed to +# re-run checks is isolated from the PR gate. +# +# A re-run replays the original run's commit, so a pull request whose last +# milestone check predates a change to the check itself replays the older +# version and keeps its stale result. See the LIMITATION section in +# .github/scripts/recheck-milestones-for-release-branch.sh. +# +# See .github/scripts/recheck-milestones-for-release-branch.sh for the details. +# +################################################################################# + +name: Recheck Milestones + +# 'create' has no branch filter, so every branch creation starts a run of this +# workflow. The guard below skips the job for anything but release/*, which is +# why this is kept out of check-milestone.yml. +on: [create] + +jobs: + recheck-open-prs: + name: Re-check open PRs after a release branch is cut + if: github.event.ref_type == 'branch' && startsWith(github.event.ref, 'release/') + runs-on: ubuntu-latest + permissions: + # 'actions: write' is needed to re-run the affected milestone checks. + actions: write + contents: read + pull-requests: read + steps: + - name: Checkout scripts + uses: actions/checkout@v6 + with: + # A 'create' run defaults to the new branch; pin the default branch so + # the script is read from a known-good copy. + ref: ${{ github.event.repository.default_branch }} + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Re-check affected pull requests + env: + # Pass the ref via env to avoid script injection from branch names. + RELEASE_BRANCH: ${{ github.event.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + WORKFLOW_FILE: check-milestone.yml + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/recheck-milestones-for-release-branch.sh" diff --git a/.github/workflows/remove-author-attention-label.yml b/.github/workflows/remove-author-attention-label.yml new file mode 100644 index 0000000000..808de9b9c2 --- /dev/null +++ b/.github/workflows/remove-author-attention-label.yml @@ -0,0 +1,83 @@ +name: Remove Author attention needed Label + +on: + issue_comment: + types: [created] + +jobs: + remove-label: + # Only run on PR comments with '/ready' from the PR author + if: >- + github.repository == 'dotnet/SqlClient' && + github.event.issue.pull_request != null && + github.event.comment.user.login == github.event.issue.user.login && + startsWith(github.event.comment.body, '/ready') + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Remove 'Author attention needed' label + uses: actions/github-script@v9 + with: + script: | + const labelName = 'Author attention needed'; + const issue_number = context.issue.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + + // Check if the label exists on the PR + const labels = await github.paginate(github.rest.issues.listLabelsOnIssue, { + owner, + repo, + issue_number, + per_page: 100, + }); + + const hasLabel = labels.some(label => label.name === labelName); + + if (!hasLabel) { + core.info(`PR #${issue_number} does not have the '${labelName}' label. No action taken.`); + return; + } + + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number, + name: labelName, + }); + core.info(`Removed '${labelName}' label from PR #${issue_number}`); + + // Re-request reviews from existing reviewers (paginate to include all reviews) + const reviews = await github.paginate(github.rest.pulls.listReviews, { + owner, + repo, + pull_number: issue_number, + per_page: 100, + }); + + // Collect unique reviewers (exclude the PR author) + const author = context.payload.issue.user.login; + const reviewers = [...new Set( + reviews + .filter(r => r.user?.type === 'User') + .map(r => r.user?.login) + .filter(login => login && login !== author) + )]; + + if (reviewers.length > 0) { + try { + await github.rest.pulls.requestReviewers({ + owner, + repo, + pull_number: issue_number, + reviewers: reviewers.slice(0, 15), + }); + core.info(`Re-requested reviews from: ${reviewers.join(', ')}`); + } catch (err) { + core.warning(`Failed to re-request reviewers: ${err.message}`); + } + } else { + core.info('No previous reviewers to re-request.'); + } diff --git a/.github/workflows/verify-aw-lock.yml b/.github/workflows/verify-aw-lock.yml new file mode 100644 index 0000000000..eb3cc2e608 --- /dev/null +++ b/.github/workflows/verify-aw-lock.yml @@ -0,0 +1,32 @@ +name: Verify gh aw lock files + +on: + pull_request: + paths: + - '.github/workflows/**/*.md' + - '.github/workflows/**/*.lock.yml' + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install gh-aw extension + uses: github/gh-aw-actions/setup-cli@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 + with: + version: v0.88.2 + + - name: Recompile agentic workflows + run: gh aw compile + + - name: Fail if any .lock.yml is out of date + run: | + if ! git diff --exit-code -- '.github/workflows/**/*.lock.yml'; then + echo "::error::One or more .github/workflows/**/*.lock.yml files are out of date relative to their .md source (running 'gh aw compile' produced a diff)." + echo "::error::Run 'gh aw compile' locally and commit the regenerated .lock.yml files in this PR." + exit 1 + fi diff --git a/.gitignore b/.gitignore index 0d8dfed6d5..9289d0ab5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,4 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore +## .gitignore for Microsoft.Data.SqlClient # User-specific files *.rsuser @@ -10,12 +7,6 @@ *.userosscache *.sln.docstates -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - # Build results [Dd]ebug/ [Dd]ebugPublic/ @@ -29,14 +20,11 @@ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ -.nuget/ -# Visual Studio 2015/2017 cache/options directory +# Visual Studio cache/options directory .vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ -# Visual Studio 2017 auto generated files +# Visual Studio auto generated files Generated\ Files/ **/.AssemblyAttributes @@ -44,34 +32,21 @@ Generated\ Files/ .vscode/* !.vscode/mcp.json -# MSTest test Results +# MSTest test results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - # TRX format test results **/*.trx -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results +# Benchmark results BenchmarkDotNet.Artifacts/ -# .NET Core +# .NET project lock files and build artifacts project.lock.json project.fragment.lock.json artifacts/ -# StyleCop -StyleCopReport.xml - # Files built by Visual Studio *_i.c *_p.c @@ -101,139 +76,36 @@ StyleCopReport.xml *.svclog *.scc -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in +# ReSharper _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user -# JustCode is a .NET coding add-in -.JustCode - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - # Visual Studio code coverage results *.coverage *.coveragexml -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages +# NuGet packages *.nupkg -# NuGet Symbol Packages *.snupkg -# Most of the packages folder can be ignored. **/[Pp]ackages/* !**/[Pp]ackages/.gitkeep +.nuget/ -# Most of the output folder can be ignored. +# Output folder **/[Oo]utput/* !**/[Oo]utput/.gitkeep -# NuGet v3's project.json files produces more ignorable files +# NuGet v3 project.json auxiliary files *.nuget.props *.nuget.targets -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - # Visual Studio cache files -# files ending in .cache can be ignored *.[Cc]ache -# but keep track of directories ending in .cache +# but keep directories ending in .cache !?*.[Cc]ache/ -# Others -ClientBin/ +# Temp and backup files ~$* *~ *.dbmdl @@ -241,138 +113,40 @@ ClientBin/ *.jfm *.pfx *.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files +# SQL Server data files *.mdf *.ldf *.ndf -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - # Microsoft Fakes FakesAssemblies/ -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat +# Node modules node_modules/ -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log +# MSBuild binary and structured log *.binlog -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - # Local History for Visual Studio .localhistory/ -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# JetBrains Rider (cross platform .NET IDE) working folder +# JetBrains Rider .idea/ -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Nuget package files -.nuget/ - # Config Json file **/config.json +**/config.jsonc # Generated Milestone PR metadata files .milestone-prs/ # MDS "Not Supported" GenAPI code **/notsupported/*.cs + +# C# language server cache +*.lscache + +# Python bytecode caches +__pycache__/ +*.py[cod] diff --git a/.vscode/mcp.json b/.vscode/mcp.json deleted file mode 100644 index a3f5378943..0000000000 --- a/.vscode/mcp.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "servers": { - "ado": { - "args": [ - "-y", - "@azure-devops/mcp", - "SqlClientDrivers" - ], - "command": "npx", - "type": "stdio" - }, - "bluebird_ctaip": { - "headers": { - "x-mcp-ec-branch": "certAuth", - "x-mcp-ec-organization": "sqlclientdrivers", - "x-mcp-ec-project": "ADO.NET", - "x-mcp-ec-repository": "Microsoft.Data.SqlClient.Ctaip" - }, - "type": "http", - "url": "https://mcp.bluebird-ai.net/" - }, - "bluebird_onebranchwiki": { - "headers": { - "x-mcp-ec-branch": "main", - "x-mcp-ec-organization": "onebranch", - "x-mcp-ec-project": "OneBranch Customer Wiki", - "x-mcp-ec-repository": "OneBranch-Customer-Wiki.v2" - }, - "type": "http", - "url": "https://mcp.bluebird-ai.net/" - }, - "bluebird-mcp-1es-docs": { - "headers": { - "x-mcp-ec-branch": "main", - "x-mcp-ec-organization": "mseng", - "x-mcp-ec-project": "1ES", - "x-mcp-ec-repository": "1ES-on-EngHub" - }, - "type": "http", - "url": "https://mcp.bluebird-ai.net/" - }, - "bluebird-mcp-sni": { - "headers": { - "x-mcp-ec-branch": "master", - "x-mcp-ec-organization": "SqlClientDrivers", - "x-mcp-ec-project": "ADO.NET", - "x-mcp-ec-repository": "Microsoft.Data.SqlClient.SNI" - }, - "type": "http", - "url": "https://mcp.bluebird-ai.net/" - }, - "bluebird-mcp-sqlclient": { - "headers": { - "x-mcp-ec-branch": "internal/main", - "x-mcp-ec-organization": "SqlClientDrivers", - "x-mcp-ec-project": "ADO.NET", - "x-mcp-ec-repository": "dotnet-sqlclient" - }, - "type": "http", - "url": "https://mcp.bluebird-ai.net/" - }, - "github": { - "type": "http", - "url": "https://api.githubcopilot.com/mcp/" - }, - "icm": { - "type": "http", - "url": "https://icm-mcp-prod.azure-api.net/v1/" - }, - "microsoft-learn": { - "type": "http", - "url": "https://learn.microsoft.com/api/mcp" - } - } -} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 56d7555c99..c461eec189 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,6 +5,7 @@ This document provides guidance for AI coding agents working with the Microsoft. ## Quick Start ### Essential Context Files + Before making changes, agents should be aware of: | File | Purpose | @@ -15,6 +16,7 @@ Before making changes, agents should be aware of: | [.github/copilot-instructions.md](.github/copilot-instructions.md) | Copilot-specific instructions | ### Detailed Technical Instructions + The `.github/instructions/` directory contains comprehensive guides: | Guide | Coverage | @@ -29,6 +31,7 @@ The `.github/instructions/` directory contains comprehensive guides: | [features.instructions.md](.github/instructions/features.instructions.md) | Feature reference, keywords | | [documentation.instructions.md](.github/instructions/documentation.instructions.md) | Documentation and samples | | [external-resources.instructions.md](.github/instructions/external-resources.instructions.md) | Docs links, version matrix, external references | +| [ado-work-items-markdown.instructions.md](.github/instructions/ado-work-items-markdown.instructions.md) | Ensure Azure DevOps work item descriptions are Markdown and preserve newlines | ## Workflow Prompts @@ -54,17 +57,40 @@ This repository provides reusable prompts in `.github/prompts/` for common maint 6. **Performance Optimization**: Use pooling, async, efficient allocations 7. **Observability**: EventSource tracing, meaningful errors +## Terminal Reliability Rules + +When using shell/terminal tools, follow these rules strictly: + +1. Treat non-zero terminal exit codes as immediate failures to investigate; do not continue as if the command succeeded. +2. If a bash session exits, assume it is dead and start a new command/session; do not wait for additional output from that session. +3. After any command expected to gather data, verify output was actually returned before proceeding. +4. If command execution failed, report the failure clearly and retry with a corrected command instead of waiting. +5. Avoid `set -e` in this automation context; prefer single-purpose commands with explicit follow-up checks so failures are visible without killing the shell unexpectedly. +6. Prefer shorter command batches over long chained scripts when collecting evidence; this makes bash exits easier to detect and recover from. + +## Branch Naming + +All branches created by AI agents **must** live under the `dev/automation/` prefix. Use a descriptive suffix, for example: + +- `dev/automation/fix-connection-timeout` +- `dev/automation/add-json-type-tests` + +Do **not** create branches directly under `main`, `dev/`, or any other top-level prefix. + ## Common Tasks ### Bug Fix Workflow + 1. Understand the issue from the bug report 2. Locate relevant code in `src/Microsoft.Data.SqlClient/src/` (do NOT modify legacy `netcore/src/` or `netfx/src/`) -3. Write a failing test that reproduces the issue -4. Implement the fix -5. Ensure all tests pass -6. Update documentation if behavior changes +3. Check `.github/instructions/features.instructions.md` for existing AppContext switches (including failover compatibility switches) before introducing behavior changes +4. Write a failing test that reproduces the issue +5. Implement the fix +6. Ensure all tests pass +7. Update documentation if behavior changes ### Feature Implementation + 1. Review the feature specification 2. Plan the implementation (see `implement-feature` prompt) 3. Update reference assemblies if adding public APIs @@ -73,6 +99,7 @@ This repository provides reusable prompts in `.github/prompts/` for common maint 6. Do not edit `CHANGELOG.md` directly; instead, add a suggested release-note entry (per `.github/copilot-instructions.md`) in the PR description or via the release-notes workflow/prompt. ### Adding Connection String Keywords + 1. Add to `SqlConnectionStringBuilder` 2. Update connection string parser 3. Default to backward-compatible value @@ -80,6 +107,7 @@ This repository provides reusable prompts in `.github/prompts/` for common maint 5. Document in feature reference ### Protocol Changes + 1. Reference MS-TDS specification 2. Update `TdsEnums.cs` for new constants 3. Implement in `TdsParser.cs` and related files @@ -87,6 +115,7 @@ This repository provides reusable prompts in `.github/prompts/` for common maint 5. Consider backward compatibility ### Performance Optimization + 1. Profile the issue using benchmarks or traces 2. Identify allocation hotspots (see `perf-optimization` prompt) 3. Apply patterns: `ArrayPool`, `Span`, static/cached instances, source generation @@ -96,6 +125,7 @@ This repository provides reusable prompts in `.github/prompts/` for common maint ### Key Documentation Links + - [Microsoft.Data.SqlClient on Microsoft Learn](https://learn.microsoft.com/sql/connect/ado-net/introduction-microsoft-data-sqlclient-namespace) - [MS-TDS Protocol Specification](https://learn.microsoft.com/openspecs/windows_protocols/ms-tds) - [SQL Server Documentation](https://learn.microsoft.com/sql/sql-server/) @@ -103,6 +133,7 @@ This repository provides reusable prompts in `.github/prompts/` for common maint ## Repository Policies See the `policy/` directory for: + - [coding-best-practices.md](policy/coding-best-practices.md) - Programming standards - [coding-style.md](policy/coding-style.md) - Code formatting guidelines - [review-process.md](policy/review-process.md) - PR review requirements diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 8f13730f68..72daa88ea8 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -1,224 +1,378 @@ -# Guidelines for Building Microsoft.Data.SqlClient + -This document provides all the necessary details to build the driver and run tests present in the repository. +# Build Guide for Microsoft.Data.SqlClient and Related Packages + +This document provides details on how to build the Microsoft.Data.SqlClient package and the other related packages +contained within this repository. ## Prerequisites ### .NET SDK -The projects in this repo require the .NET 10.0 SDK to build. Please ensure you -have the latest version of that SDK installed. - -Tests and tools may require different .NET Runtimes that may be installed -independently. For example, tests targeting .NET 8.0 will need that runtime -installed. - -### Visual Studio - -This project should be built with Visual Studio 2019+ for the best compatibility. The required set of components are provided in the below file: - -- **Visual Studio 2019** with imported components: [VS19Components](/tools/vsconfig/VS19Components.vsconfig) - -- **Powershell**: To build SqlClient on Linux, powershell is needed as well. Follow the distro specific instructions at [Install Powershell on Linux](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-linux?view=powershell-7.4) - -Once the environment is setup properly, execute the desired set of commands below from the _root_ folder to perform the respective operations: - -### Manual Test Prerequisites - -Manual Tests require the below setup to run: - -- SQL Server with enabled Shared Memory, TCP and Named Pipes Protocols and access to the Client OS. -- Databases "NORTHWIND" and "UdtTestDb" present in SQL Server, created using SQL scripts [createNorthwindDb.sql](tools/testsql/createNorthwindDb.sql) and [createUdtTestDb.sql](tools/testsql/createUdtTestDb.sql). To setup an Azure Database with "NORTHWIND" tables, use SQL Script: [createNorthwindAzureDb.sql](tools/testsql/createNorthwindAzureDb.sql). -- Make a copy of the configuration file [config.default.json](src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.json) and rename it to `config.json`. Update the values in `config.json`: - - |Property|Description|Value| - |------|--------|-------------------| - |TCPConnectionString | Connection String for a TCP enabled SQL Server instance. | `Server={servername};Database={Database_Name};Trusted_Connection=True;`
OR `Data Source={servername};Initial Catalog={Database_Name};Integrated Security=True;`| - |NPConnectionString | Connection String for a Named Pipes enabled SQL Server instance.| `Server=\\{servername}\pipe\sql\query;Database={Database_Name};Trusted_Connection=True;`
OR
`Data Source=np:{servername};Initial Catalog={Database_Name};Integrated Security=True;`| - |TCPConnectionStringHGSVBS | (Optional) Connection String for a TCP enabled SQL Server with Host Guardian Service (HGS) attestation protocol configuration. | `Server=tcp:{servername}; Database={Database_Name}; UID={UID}; PWD={PWD}; Attestation Protocol = HGS; Enclave Attestation Url = {AttestationURL};`| - |TCPConnectionStringNoneVBS | (Optional) Connection String for a TCP enabled SQL Server with a VBS Enclave and using None Attestation protocol configuration. | `Server=tcp:{servername}; Database={Database_Name}; UID={UID}; PWD={PWD}; Attestation Protocol = NONE;`| - |TCPConnectionStringAASSGX | (Optional) Connection String for a TCP enabled SQL Server with a SGX Enclave and using Microsoft Azure Attestation (AAS) attestation protocol configuration. | `Server=tcp:{servername}; Database={Database_Name}; UID={UID}; PWD={PWD}; Attestation Protocol = AAS; Enclave Attestation Url = {AttestationURL};`| - |EnclaveEnabled | Enables tests requiring an enclave-configured server.| - |TracingEnabled | Enables EventSource related tests | - |AADAuthorityURL | (Optional) Identifies the OAuth2 authority resource for `Server` specified in `AADPasswordConnectionString` | `https://login.windows.net/`, where `` is the tenant ID of the Entra ID (Azure AD) tenant | - |AADPasswordConnectionString | (Optional) Connection String for testing Entra ID Password Authentication. | `Data Source={server.database.windows.net}; Initial Catalog={Azure_DB_Name};Authentication=Active Directory Password; User ID={AAD_User}; Password={AAD_User_Password};`| - |AADSecurePrincipalId | (Optional) The Application Id of a registered application which has been granted permission to the database defined in the AADPasswordConnectionString. | {Application ID} | - |AADSecurePrincipalSecret | (Optional) A Secret defined for a registered application which has been granted permission to the database defined in the AADPasswordConnectionString. | {Secret} | - |AzureKeyVaultURL | (Optional) Azure Key Vault Identifier URL | `https://{keyvaultname}.vault.azure.net/` | - |AzureKeyVaultTenantId | (Optional) The Entra ID tenant (directory) Id of the service principal. | _{Tenant ID of Active Directory}_ | - |SupportsIntegratedSecurity | (Optional) Whether or not the USER running tests has integrated security access to the target SQL Server.| `true` OR `false`| - |LocalDbAppName | (Optional) If Local Db Testing is supported, this property configures the name of Local DB App instance available in client environment. Empty string value disables Local Db testing. | Name of Local Db App to connect to.| - |LocalDbSharedInstanceName | (Optional) If LocalDB testing is supported and the instance is shared, this property configures the name of the shared instance of LocalDB to connect to. | Name of shared instance of LocalDB. | - |FileStreamDirectory | (Optional) If File Stream is enabled on SQL Server, pass local directory path to be used for setting up File Stream enabled database. | `D:\\escaped\\absolute\\path\\to\\directory\\` | - |UseManagedSNIOnWindows | (Optional) Enables testing with Managed SNI on Windows| `true` OR `false`| - |DNSCachingConnString | Connection string for a server that supports DNS Caching| - |EnclaveAzureDatabaseConnString | (Optional) Connection string for Azure database with enclaves | - |ManagedIdentitySupported | (Optional) When set to `false` **Managed Identity** related tests won't run. The default value is `true`. | - |IsManagedInstance | (Optional) When set to `true` **TVP** related tests will use non-Azure bsl files to compare test results. This is needed when testing against Azure Managed Instances; otherwise TVP Tests will fail on TestSet 3. The default value is `false`. | - |PowerShellPath | The full path to PowerShell.exe. This is not required if the path is present in the PATH environment variable. | `D:\\escaped\\absolute\\path\\to\\PowerShell.exe` | - -## MSBuild Reference - -### Targets - -The following build targets are defined in `build.proj`: - -|Target|Description| -|-|-| -|`BuildAbstractions`|Restore and build the Abstractions package.| -|`BuildAkvProvider`|Builds the Azure Key Vault Provider package for all supported platforms.| -|`BuildAllConfigurations`|Default target. Builds the .NET Framework and .NET drivers for all target frameworks and operating systems.| -|`BuildAzure`|Restore and build the Azure package.| -|`BuildLogging`|Restore and build the Logging package.| -|`BuildNetCore`|Builds the .NET driver for all target frameworks.| -|`BuildNetCoreAllOS`|Builds the .NET driver for all target frameworks and operating systems.| -|`BuildNetFx`|Builds the .NET Framework driver for all target frameworks.| -|`BuildSqlClient`|Build the driver for all target frameworks.| -|`Clean`|Cleans all generated files.| -|`PackAbstractions`|Pack the Abstractions NuGet package into `packages/`. Requires `BuildAbstractions` first.| -|`PackAkvProvider`|Pack the Azure Key Vault Provider NuGet package (requires a prior build).| -|`PackAzure`|Pack the Azure NuGet package into `packages/`. Requires `BuildAzure` first.| -|`PackLogging`|Pack the Logging NuGet package into `packages/`. Requires `BuildLogging` first.| -|`Restore`|Restores NuGet packages.| -|`RunTests`|Runs the unit, functional, and manual tests for the .NET Framework and .NET drivers| -|`RunUnitTests`|Runs just the unit tests for the .NET Framework and .NET drivers| -|`RunFunctionalTests`|Runs just the functional tests for the .NET Framework and .NET drivers| -|`RunManualTests`|Runs just the manual tests for the .NET Framework and .NET drivers| - -### Parameters - -The following parameters may be defined as MSBuild properties to configure the -build: - -|Name|Supported Values|Default|Description| -|-|-|-|-| -|`Configuration`|`Debug`, `Release`|`Debug`|Sets the release configuration.| -|`OSGroup`|`Unix`, `Windows_NT`, `AnyOS`|typically defaults to the client system's OS, unless using `BuildAllConfigurations` or an `AnyOS` specific target|The operating system to target.| -|`Platform`|`AnyCPU`, `x86`, `x64`, `ARM`, `ARM64`|`AnyCPU`|May only be set when using package reference type or running tests.| -|`TestSet`|`1`, `2`, `3`, `AE`, or any combination thereof|`''`|Build or run a subset of the manual tests. Omit (default) to run all tests.| -|`DotnetPath`|Absolute file path to an installed `dotnet` version.|The system default specified by the path variable|Set to run tests using a specific dotnet version (e.g. C:\net6-win-x86\)| -|`TF`|`net8.0`, `net462`, `net47`, `net471`, `net472`, `net48`, `net481`|`net9.0` in netcore, `net462` in netfx|Sets the target framework when building or running tests. Not applicable when building the drivers.| -|`ResultsDirectory`|An absolute file path|./TestResults relative to current directory|Specifies where to write test results.| - -## Example Commands to Run Tests Using MSBuild (Recommended) - -Using the default configuration and running all tests: +Projects in this repository require the .NET SDK to be installed in order to build. For the exact version required for +building the current version, see [global.json](global.json). Downloads for .NET SDK can be found at +[.NET Downloads](https://dotnet.microsoft.com/en-us/download/dotnet). + +The .NET SDK contains support for building for previous versions of .NET, including support for building .NET Framework +on operating systems that do not support .NET Framework. As such, it is not necessary to install any version of the +.NET SDK aside from the version specified in [global.json](global.json). + +### Miscellaneous + +**PowerShell** is included as a .NET local tool in this repository. Running `dotnet tool restore` +(see below) will make it available via `dotnet tool run pwsh -- `. Note that `pwsh` is not +added to PATH — it must be invoked through `dotnet tool run`. Build targets handle this +automatically; manual invocation is only needed for ad-hoc scripting. + +The **NuGet** binary is optional for inspection and feed-management workflows, but build and packaging flows in this +repository are run through `dotnet build` against `build.proj`. + +### .NET Tools + +This repository uses .NET local tools (e.g. PowerShell) that must be restored before building. Run the following from the repository root: ```bash -msbuild -t:RunTests +dotnet tool restore ``` -Using the Release configuration: +## Developer Workflow + +Once you've cloned the repository and made your changes to the codebase, it is time to build, test, and optionally +package the project. The `build.proj` file provides convenient targets to accomplish these tasks. + +> [!NOTE] +> Although every effort has been made to make building and testing work in your IDE of choice, some quirks in behavior +> may be noticed, possibly severe. All official build and test infrastructure uses the `build.proj` entrypoint, and it +> is recommended that `build.proj` is used for local development, as well. + + + +> [!TIP] +> This section is not exhaustive of all targets or parameters to `build.proj`. Complete documentation is available in +> [`build.proj`](build.proj). + +### Building Projects + +From the root of your repository, run `dotnet build` against `build.proj` with a build target, following this pattern: ```bash -msbuild -t:RunTests -p:Configuration=Release +dotnet build build.proj -t: [optional_parameters] ``` -Running only the unit tests: +Since `build.proj` is the only project file in the repo root, it can be omitted when building from +the root: ```bash -msbuild -t:RunUnitTests +dotnet build -t: [optional_parameters] ``` -Using a specific .NET runtime to run tests: +The command-line examples below will assume that `build.proj` is selected by default and will omit +it from the `dotnet build` command. + +If no target is specified, `build.proj` runs the `BuildAll` target by default, which builds all +projects, tests, samples, and tools for all supported OS combinations. To build only the driver +projects, specify `-t:BuildDriver` explicitly. + +The following build targets can be used to build the following projects. All targets will implicitly build any other +projects they depend on. + +| `` | Description | +|-------------------------------|---------------------------------------------------------------------------------| +| `BuildAbstractions` | Builds Microsoft.Data.SqlClient.Extensions.Abstractions | +| `BuildAkvProvider` | Builds Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider | +| `BuildAll` | Builds all projects, tests, samples, and tools for all supported OS combinations (default target) | +| `BuildAzure` | Builds Microsoft.Data.SqlClient.Extensions.Azure | +| `BuildDriver` | Builds all driver projects for all platforms | +| `BuildLogging` | Builds Microsoft.Data.SqlClient.Internal.Logging | +| `BuildSamples` | Builds the sample projects under `doc/samples/` | +| `BuildSqlClient` | Builds all variants of Microsoft.Data.SqlClient, for all platforms | +| `BuildSqlClientNotSupported` | Builds the "unsupported platform" assemblies for Microsoft.Data.SqlClient | +| `BuildSqlClientRef` | Builds the reference assemblies for Microsoft.Data.SqlClient | +| `BuildSqlClientUnix` | Builds the Unix-specific implementation binaries of Microsoft.Data.SqlClient | +| `BuildSqlClientWindows` | Builds the Windows-specific implementation binaries of Microsoft.Data.SqlClient | +| `BuildSqlServer` | Builds Microsoft.SqlServer.Server | +| `BuildTests` | Builds all test projects for all supported OS combinations | +| `BuildTools` | Builds auxiliary tool/app projects and their test projects | +| `Clean` | Removes build and test output directories | + +A selection of parameters for build targets in `build.proj` can be found below: + + + +| `[optional_parameter]` | Allowed Values | Default | Description | +|-----------------------------------|----------------------------------|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `-p:Configuration=` | `Debug`, `Release` | `Debug` | Build configuration | +| `-p:PackageVersionSqlClient=` | `major.minor.patch[-prerelease]` | `[blank]` | Version to assign to the SqlClient family (`Microsoft.Data.SqlClient`, `Internal.Logging`, `Extensions.Abstractions`, `Extensions.Azure`, and the AKV Provider all share it). Assembly and file versions are derived from this, if it is provided. See Versioning for more details | +| `-p:PackageVersionSqlServer=` | `major.minor.patch[-prerelease]` | `[blank]` | Version to assign to `Microsoft.SqlServer.Server`, which is versioned separately from the SqlClient family. | + + + +For most projects, build output is placed in `artifacts//Project-/`. `` +is the full name of the package, `` is the build configuration, and `` is the target framework +moniker. SqlClient deviates slightly from this convention, since it consists of multiple projects and the +implementation project is OS-specific. Implementation project output is placed in +`artifacts/Microsoft.Data.SqlClient/Project-//`. The unsupported platform assemblies are placed +in `artifacts/Microsoft.Data.SqlClient.unsupported/Project-/`, and the reference assemblies are +placed in `artifacts/Microsoft.Data.SqlClient.ref/Project-/`. + +#### Examples + +Build everything (all projects, tests, samples, and tools) using the default target: ```bash -msbuild -t:RunTests -p:DotnetPath=C:\net8-win-x86\ +dotnet build ``` -To run tests against a specific version of .NET/.NET Framework, set the `-p:TF` parameter. +Build only the driver projects: ```bash -msbuild -t:RunTests -p:TF=net8.0 -msbuild -t:RunTests -p:TF=net462 +dotnet build -t:BuildDriver ``` -## Example Commands to Run Tests using `dotnet` +Build Microsoft.Data.SqlClient in Release configuration: + +```bash +dotnet build -t:BuildSqlClient -p:Configuration=Release +``` -Under the hood, the MSBuild commands to run tests use `dotnet` commands. But, if you wish to run -them without the overhead of wrapping/unwrapping in MSBuild, you can run them directly. +Build a specific version of Microsoft.Data.SqlClient.Extensions.Abstractions (Abstractions is part of the +SqlClient family, so its version is set via the family parameter `PackageVersionSqlClient`): -To change the processor architecture that runs the test (where possible, ie, x86 on x64), use the -appropriate `dotnet` executable. +```bash +dotnet build -t:BuildAbstractions -p:PackageVersionSqlClient=7.1.0 +``` -By default, the tests will be executed on all supported .NET/.NET framework versions. To run on a -specific version, pass the `-f` parameter with the desired version (eg `net9.0`). +### Testing Projects -The `--filter` parameter is used to select which tests run. The default `category!=failing& -category!=flaky&category!=interactive` prevents tests that are known to be failing or flaky from -running. To run a specific test, use `FullyQualifiedName=[fully qualified name of the test method]` -as the filter parameter. To run all possible tests, even known failing and flaky ones, simply omit -the filter parameter. Please note, however, that this will still omit tests that cannot run on the -current platform or with the current test configuration (eg, Windows tests on Linux, or SQL DB tests -when Azure Synapse is configured). +This section provides a summary and brief example of how to execute tests for projects in this repository. **For more +information about test procedures, including config file setup, see [TESTGUIDE.md](TESTGUIDE.md).** -### Run Functional Tests +From the root of your repository, run `dotnet build` against `build.proj` with a test target, following this pattern: ```bash -dotnet test "src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj" \ - -p:Configuration=Release \ - --filter "category!=failing&category!=flaky&category!=interactive" +dotnet build -t: [optional_parameters] +``` + +| `` | Description | +|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `Test` | Runs all tests in the repository for all platforms supported by the host OS. _This will take a considerable amount of time and is not recommended_. | +| `TestAbstractions` | Runs all tests for Microsoft.Data.SqlClient.Extensions.Abstractions | +| `TestAkvProvider` | Runs the unit test project for Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider. | +| `TestAzure` | Runs all tests for Microsoft.Data.SqlClient.Extensions.Azure | +| `TestSqlClient` | Runs all tests for Microsoft.Data.SqlClient. | +| `TestSqlClientFunctional` | Runs the "functional" test project for Microsoft.Data.SqlClient. These are a mix of unit and integration tests against live servers. | +| `TestSqlClientManual` | Runs the "manual" test project for Microsoft.Data.SqlClient. These are generally integration tests against live servers. | +| `TestSqlClientUnit` | Runs the unit test project for Microsoft.Data.SqlClient. These are a mix of unit tests and integration tests against simulated servers. | + +> [!TIP] +> Test targets will automatically build the projects they depend on. Therefore, it is not necessary to explicitly build +> (eg) SqlClient before executing the (eg) functional tests target. + +A selection of parameters for test targets in `build.proj` relevant to common developer workflows can be found below: + + +| `[optional_parameter]` | Default Value | Description | +|------------------------|----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-p:Configuration=` | `Debug` | Build configuration. Can be `Debug` or `Release`. | +| `-p:DotnetPath=` | `[blank]` | Path to `dotnet` binary to run the test project. This is useful for running tests against x86 platform on a x86_64 machine. Path must end with `\` or `/`. | +| `-p:TestBlameTimeout=` | `10m` | How long to wait on a test before timing it out. Use `0` to disable hang timeouts. | +| `-p:TestFilters=` | `category!=failing&category!=flaky&category!=interactive` | Filters to use to select the xUnit tests to execute. Use `none` to run all possible tests. | +| `-p:TestFramework=` | `[blank]` | Target framework moniker for the version of .NET to use to execute tests. | +| `-p:TestSet=` | `[blank]` | The `TestSqlClientManual` project is very large and is split into multiple sets that can be executed individually. This parameter allows selecting between test sets: `1`, `2`, `3`, and `AE`. | + + + +#### Examples + +Run Microsoft.Data.SqlClient unit tests: + +```bash +dotnet build -t:TestSqlClientUnit ``` -### Run Manual Tests +Run Microsoft.Data.SqlClient manual test set 2: ```bash -dotnet test "src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj" \ - -p:Configuration=Release \ - --filter "category!=failing&category!=flaky&category!=interactive" +dotnet build -t:TestSqlClientManual -p:TestSet=2 ``` -### Run Unit Tests +Run Microsoft.Data.SqlClient functional tests against x86 dotnet: + ```bash -dotnet test "src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj" \ - -p:Configuration=Release \ - --filter "category!=failing&category!=flaky&category!=interactive" +dotnet build -t:TestSqlClientFunctional -p:DotnetPath='C:\path\to\dotnet\x86\' ``` -## Testing with Package References +Run all Microsoft.Data.SqlClient.Extensions.Azure unit tests, including interactive, but excluding failing tests: -The MDS driver consists of several components, each of which produces its own -NuGet package. During development, components reference each other via -`` properties by default. This means that building -and testing one component will implicitly build its project referenced -dependencies. +```bash +dotnet build -t:TestAzure -p:TestFilters=category!=failing +``` + +Run Microsoft.Data.SqlClient functional tests against net8.0 runtime: + +```bash +dotnet build -t:TestSqlClientFunctional -p:TestFramework=net8.0 +``` -Alternatively, the `ReferenceType` build property may be specified with a value -of `Package`. This will change inter-component dependencies to use -`` dependencies, and require that dependent components be -built and packaged before building the depending component. This will generate NuGet -packages in the root packages/ directory, and will be automatically searched by NuGet -(see our root `NuGet.config`). +### Packaging Projects -Then, you can specify `Package` references be used, for example: +Just like building and testing the various projects in this repository, packaging the projects into NuGet packages is +also handled by `build.proj`. From the root of your repository, run `dotnet build` against `build.proj` with a pack target, +following this pattern: ```bash -dotnet build -t:BuildLogging,PackLogging -dotnet build -t:BuildSqlServer,PackSqlServer -dotnet build -t:BuildAbstractions,PackAbstractions -p:ReferenceType=Package -dotnet build -t:BuildAzure,PackAzure -p:ReferenceType=Package -dotnet build -t:BuildSqlClient -p:ReferenceType=Package -dotnet build -t:GenerateMdsPackage -dotnet build -t:BuildAKVNetCore -p:ReferenceType=Package -dotnet build -t:GenerateAkvPackage +dotnet build -t: [optional_parameters] ``` -The above will build the MDS and AKV components, place their NuGet packages into -the `packages/` directory. +| `` | Description | +|--------------------|-------------------------------------------------------------------------------------| +| `Pack` | Packages all projects in the repository. | +| `PackAbstractions` | Packages the Microsoft.Data.SqlClient.Extensions.Abstractions package | +| `PackAkvProvider` | Packages the Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider package | +| `PackAzure` | Packages the Microsoft.Data.SqlClient.Extensions.Azure package | +| `PackLogging` | Packages the Microsoft.Data.SqlClient.Internal.Logging package | +| `PackSqlClient` | Packages the Microsoft.Data.SqlClient package | +| `PackSqlServer` | Packages the Microsoft.SqlServer.Server package | + +> [!TIP] +> For convenience, the Pack targets will automatically build the target project and any dependencies. + +A selection of parameters for pack targets in `build.proj` relevant to common developer workflows can be found below: + + + +| `[optional_parameter]` | Default Value | Allowed Values | Description | +|------------------------------------|---------------|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `-p:Configuration=` | `Debug` | `Debug`, `Release` | Build configuration. Only applies if project and dependencies are being built. | +| `-p:PackBuild=` | `true` | `true`, `false` | Whether or not to build the project before packing. If `false`, project must be built using the same parameters. | +| `-p:PackageVersionSqlClient=` | `[blank]` | eg. `7.1.0-dev123` | Version to assign to the entire SqlClient family (`Microsoft.Data.SqlClient`, `Internal.Logging`, `Extensions.Abstractions`, `Extensions.Azure`, and the AKV Provider — they all share the SqlClient version). If `PackBuild` is `true`, the assembly and file versions are derived from this version. See Versioning for more details. | +| `-p:PackageVersionSqlServer=` | `[blank]` | eg. `1.1.0-dev123` | Version to assign to `Microsoft.SqlServer.Server`, which is versioned separately from the SqlClient family. | + + -A non-AnyCPU platform reference can only be used with package reference type. -Otherwise, the specified platform will be replaced with AnyCPU in the build -process. +For `PackSqlClient`, the SqlClient nuspec pins its family dependencies (Abstractions and Logging) to the same `SqlClientPackageVersion` value, so a single `-p:PackageVersionSqlClient=` controls both the SqlClient package version and those dependency ranges. `Microsoft.SqlServer.Server` is pinned separately via `-p:PackageVersionSqlServer=`. -### Running Tests with Reference Type +If omitted, `PackSqlClient` computes these versions from `Versions.props` using the current `BuildNumber` and `BuildSuffix` context. + +#### Examples + +Package Microsoft.Data.SqlClient.Internal.Logging into a NuGet package: + +```bash +dotnet build -t:PackLogging +``` -Provide property to `dotnet test` commands for testing desired reference type. +Package Microsoft.Data.SqlClient: ```bash -dotnet test -p:ReferenceType=Project ... +dotnet build -t:PackSqlClient ``` +Package a specific version of Microsoft.Data.SqlClient.Extensions.Abstractions (set via the family parameter +`PackageVersionSqlClient`): + +```bash +dotnet build -t:PackAbstractions -p:PackageVersionSqlClient=7.1.0 +``` + +Package Microsoft.Data.SqlClient.Extensions.Azure without building it beforehand: + +```bash +dotnet build -t:PackAzure -p:PackBuild=false +``` + +## Versioning + +Versioning can be accomplished by using a mix of different parameters to the `build.proj` targets: +`PackageVersionSqlClient` (or `PackageVersionSqlServer`), `BuildNumber`, and `BuildSuffix`. Using these in different +combinations can generate appropriate package, assembly, and file versions for different scenarios. For most developer +workflows, it is not necessary to specify any of these parameters - appropriate versions based on the latest release +will be generated automatically. This section primarily exists to document the various parameters, their effects, and +the scenarios they can be useful for. + +All packages in the **SqlClient family** (`Microsoft.Data.SqlClient`, `Internal.Logging`, `Extensions.Abstractions`, +`Extensions.Azure`, and the AKV Provider) share a single version, set via `-p:PackageVersionSqlClient`. +`Microsoft.SqlServer.Server` is versioned separately via `-p:PackageVersionSqlServer`. + +The SqlClient family version is defined in `src/Microsoft.Data.SqlClient/Versions.props` (and SqlServer's in its own +`Versions.props`), which declares a "default" version — the next version to release. For the table below, we assume this +is "1.2.3". + +| `PackageVersion` | `BuildNumber` | `BuildSuffix` | Package Version | Assembly Version | File Version | Scenario | +|------------------|---------------|---------------|------------------|------------------|---------------|------------------------------------------------------------| +| N/A | N/A | N/A | `1.2.3-dev` | `1.0.0` | `1.2.3.0` | Standard developer scenario | +| `9.8.7` | N/A | N/A | `9.8.7` | `9.0.0` | `9.8.7.0` | Developer is building a specific version of the package | +| `9.8.7-preview1` | N/A | N/A | `9.8.7-preview1` | `9.0.0` | `9.8.7.0` | Developer is building a pre-release version of the package | +| N/A | `1234` | N/A | `1.2.3` | `1.0.0` | `1.2.3.1234` | Automated pipelines building GA releases | +| N/A | `1234` | `ci` | `1.2.3-ci1234` | `1.0.0` | `1.2.3.1234` | Automated pipelines building non-prod releases | + +--- + +## Package Mode Builds + +The above documentation is the default mode of operation, and is the recommended mode for most developers. However, +`build.proj` supports "package mode" builds. In this mode, instead of projects depending on other projects, they +depend on NuGet packages. This mode is useful for verifying that packages work with each other, especially in automated +build scenarios. For completeness, and debugging of automated builds, this section documents behavior of "package mode". + +To switch to "package mode", set the `ReferenceType` parameter in `build.proj` to `Package`. And, optionally, include +one or both of the following parameters: + +- `PackageVersionSqlClient` — the version for the entire SqlClient family. +- `PackageVersionSqlServer` — the version for `Microsoft.SqlServer.Server`. + +These parameters pull double duty. In targets where a package is being built, the parameter sets the version of the +package. In targets where a package is being referenced, the parameter sets the version of the referenced package. +Because the SqlClient family shares one version, `PackageVersionSqlClient` covers every family package, whether it is +being built or referenced. + +If these parameters are not specified, the latest version, as defined in the `Versions.props` file, will be used. + +The `nuget.config` for this repository defines a local feed that points to the `packages` directory. This allows +developers that need to test against development packages to drop their development packages into this directory, and +run subsequent `build.proj` targets against them. + +### Examples + +Build Microsoft.Data.SqlClient version 7.1.1 in package mode. Because all SqlClient family packages share the same +version, a single `-p:PackageVersionSqlClient=7.1.1` applies to SqlClient and its family dependencies (Abstractions and +Logging). + +Build v7.1.1 of Logging and copy to packages: + +```bash +dotnet build -t:PackLogging -p:ReferenceType=Package -p:PackageVersionSqlClient=7.1.1 +cp artifacts/Microsoft.Data.SqlClient.Internal.Logging/Debug/*.*pkg packages/ +``` + +Build v7.1.1 of Abstractions (which depends on v7.1.1 of Logging): + +```bash +dotnet build -t:PackAbstractions \ + -p:ReferenceType=Package \ + -p:PackageVersionSqlClient=7.1.1 +cp artifacts/Microsoft.Data.SqlClient.Extensions.Abstractions/Package-Debug/*.*pkg packages/ +``` + +Build SqlClient: + +```bash +dotnet build -t:PackSqlClient \ + -p:ReferenceType=Package \ + -p:PackageVersionSqlClient=7.1.1 +cp artifacts/Microsoft.Data.SqlClient/Package-Debug/*.*pkg packages/ +``` + +Run Microsoft.Data.SqlClient functional tests against the versions built above: + +```bash +dotnet build -t:TestSqlClientFunctional \ + -p:ReferenceType=Package \ + -p:PackageVersionSqlClient=7.1.1 +``` + +Manual test prerequisites and configuration are covered in [TESTGUIDE.md](TESTGUIDE.md#manual-test-prerequisites). ## Using Managed SNI on Windows @@ -274,10 +428,14 @@ PowerShell: Bash: + + ```bash $ cd src/Microsoft.Data.SqlClient/tests/PerformanceTests ``` + + ### Create Database Create an empty database for the benchmarks to use. This example assumes @@ -290,13 +448,14 @@ $ sqlcmd -S localhost -U sa -P password 1> quit ``` -The default `runnerconfig.json` expects a database named `sqlclient-perf-db`, -but you may change the config to use any existing database. All tables in -the database will be dropped when running the benchmarks. +The default `runnerconfig.jsonc` expects a database named `sqlclient-perf-db`, +but you may change the config to use any existing database. The benchmarks +create and drop their own tables (typically prefixed with `perf_`) in this +database; other existing tables are left untouched. ### Configure Runner -Configure the benchmarks by editing the `runnerconfig.json` file directly in the +Configure the benchmarks by editing the `runnerconfig.jsonc` file directly in the `PerformanceTests` directory with an appropriate connection string and benchmark settings: @@ -304,6 +463,9 @@ settings: { "ConnectionString": "Server=tcp:localhost; Integrated Security=true; Initial Catalog=sqlclient-perf-db;", "UseManagedSniOnWindows": false, + "UseOptimizedAsyncBehaviour": true, + "WaitForProfiler": false, + "UseNativeMemoryAndETWProfiler": false, "Benchmarks": { "SqlConnectionRunnerConfig": @@ -323,36 +485,51 @@ settings: Individual benchmarks may be enabled or disabled, and each has several benchmarking options for fine tuning. -After making edits to `runnerconfig.json` you must perform a build which will +The top-level flags control global runner behavior: + +| Flag | Description | +| --- | --- | +| `UseManagedSniOnWindows` | Enables the managed SNI implementation on Windows instead of native SNI. | +| `UseOptimizedAsyncBehaviour` | Enables packet multiplexing and other async optimizations in SqlClient. | +| `WaitForProfiler` | Pauses at startup and prints the process ID so you can attach an external profiler (e.g. `dotnet-trace`) before benchmarks run. | +| `UseNativeMemoryAndETWProfiler` | Attaches the `NativeMemoryProfiler` and `EtwProfiler` BenchmarkDotNet diagnosers. Windows only; has no effect on other OSes. | + +Some benchmarks (e.g. `DataTypeReaderRunner`) also +read per-type test values from `datatypes.json` in the `PerformanceTests` +directory. Like `runnerconfig.jsonc`, this file's location can be overridden +with the `DATATYPES_CONFIG` environment variable. + +After making edits to `runnerconfig.jsonc` you must perform a build which will copy the file into the `artifacts` directory alongside the benchmark DLL. By -default, the benchmarks look for `runnerconfig.json` in the same directory as +default, the benchmarks look for `runnerconfig.jsonc` in the same directory as the DLL. -Optionally, to avoid polluting your git workspace and requring a build after -each config change, copy `runnerconfig.json` to a new file, make your edits +Optionally, to avoid polluting your git workspace and requiring a build after +each config change, copy `runnerconfig.jsonc` to a new file, make your edits there, and then specify the new file with the RUNNER_CONFIG environment -variable. +variable. The same approach works for `datatypes.json` via the +`DATATYPES_CONFIG` environment variable. PowerShell: ```pwsh -> copy runnerconfig.json $HOME\.configs\runnerconfig.json +> copy runnerconfig.jsonc $HOME\.configs\runnerconfig.jsonc -# Make edits to $HOME\.configs\runnerconfig.json +# Make edits to $HOME\.configs\runnerconfig.jsonc # You must set the RUNNER_CONFIG environment variable for the current shell. -> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.json" +> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.jsonc" ``` Bash: ```bash -$ cp runnerconfig.json ~/.configs/runnerconfig.json +$ cp runnerconfig.jsonc ~/.configs/runnerconfig.jsonc -# Make edits to ~/.configs/runnerconfig.json +# Make edits to ~/.configs/runnerconfig.jsonc # Optionally export RUNNER_CONFIG. -$ export RUNNER_CONFIG=~/.configs/runnerconfig.json +$ export RUNNER_CONFIG=~/.configs/runnerconfig.jsonc ``` ### Run Benchmarks @@ -372,5 +549,5 @@ Bash: # copy prepared by the build. $ dotnet run -c Release -f net9.0 -$ RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0 +$ RUNNER_CONFIG=~/.configs/runnerconfig.jsonc dotnet run -c Release -f net9.0 ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index dc51c7d2bc..3901428151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,597 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - > **Note:** Releases are sorted in reverse chronological order (newest first). +## [Stable Release 7.0.3] - 2026-09-10 + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4599](https://github.com/dotnet/SqlClient/pull/4599)) + +### Fixed + +- Fixed a `SqlBulkCopy` regression in environments where the application login cannot read `sys.all_columns`. Bulk copy now falls back to the earlier column-discovery behavior when that permission is unavailable. Support for hidden columns and SQL Graph column aliases still requires access to the metadata view. + ([#4370](https://github.com/dotnet/SqlClient/issues/4370), [#4306](https://github.com/dotnet/SqlClient/pull/4306), [#4402](https://github.com/dotnet/SqlClient/pull/4402)) + +- Fixed a memory-allocation regression in connection and command operations caused by formatting diagnostic strings even when tracing was disabled. Also corrected trace messages that reported an incorrect object ID or could throw `FormatException` when traced values contained braces. + ([#4528](https://github.com/dotnet/SqlClient/pull/4528), [#4533](https://github.com/dotnet/SqlClient/pull/4533)) + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4583](https://github.com/dotnet/SqlClient/pull/4583)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4553](https://github.com/dotnet/SqlClient/pull/4553)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561)) + +- Fixed authentication state handling so clearing `SqlConnection.AccessToken`, `AccessTokenCallback`, or `SspiContextProvider` preserves the other authentication values in the connection pool key. Cloning a connection or updating its credential also preserves its `SspiContextProvider`. Combining a non-null `SspiContextProvider` with `AccessToken` or `AccessTokenCallback` now throws `InvalidOperationException` instead of silently discarding authentication state; applications must use one authentication mechanism at a time. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561), [#4644](https://github.com/dotnet/SqlClient/pull/4644)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4663](https://github.com/dotnet/SqlClient/pull/4663)) + +### Companion packages + +- Released `Microsoft.Data.SqlClient.Extensions.Azure` 7.0.3 with the Entra ID authority parsing fix for Dataverse/Dynamics 365 connections. See [release notes](release-notes/Extensions/Azure/7.0/7.0.3.md). +- Released version-aligned `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` 7.0.3 with no functional or API changes. See the [Azure Key Vault provider](release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md), [Abstractions](release-notes/Extensions/Abstractions/7.0/7.0.3.md), and [Logging](release-notes/Internal/Logging/7.0/7.0.3.md) release notes. + +## [Stable Release 6.1.7] - 2026-09-10 + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4598](https://github.com/dotnet/SqlClient/pull/4598)) + +### Fixed + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4584](https://github.com/dotnet/SqlClient/pull/4584)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4552](https://github.com/dotnet/SqlClient/pull/4552)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed token authentication state handling so clearing `SqlConnection.AccessToken` preserves an existing `AccessTokenCallback` in the connection pool key, and clearing `AccessTokenCallback` preserves an existing `AccessToken`. Callback-based authentication now also follows the same prelogin server-certificate validation rules as an explicitly supplied access token. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4664](https://github.com/dotnet/SqlClient/pull/4664)) + +## [Preview Release 7.1.0-preview3] - 2026-08-26 + +This update brings the following changes since the [7.1.0-preview2](release-notes/7.1/7.1.0-preview2.md) release. +See the [full release notes](release-notes/7.1/7.1.0-preview3.md) for detailed descriptions. + +> **Important — package version alignment:** Starting with the [7.0.2](release-notes/7.0/7.0.2.md) release, the `Microsoft.Data.SqlClient` driver and its companion packages share a single aligned version. Preview 3 of the `7.1` line continues this alignment; the following packages now ship together as `7.1.0-preview3`: +> +> - `Microsoft.Data.SqlClient` +> - `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` +> - `Microsoft.Data.SqlClient.Extensions.Azure` +> - `Microsoft.Data.SqlClient.Extensions.Abstractions` +> - `Microsoft.Data.SqlClient.Internal.Logging` +> +> (`Microsoft.SqlServer.Server` continues to version independently and remains at `1.0.0`.) +> +> Applications that reference `Microsoft.Data.SqlClient.Extensions.Azure` must upgrade it to `7.1.0-preview3` when upgrading `Microsoft.Data.SqlClient` to `7.1.0-preview3`. +> +> **Compatibility guarantee:** All aligned assemblies ship with `FileVersion 7.1.0.x` and `AssemblyVersion 7.0.0.0`. The `AssemblyVersion` is unchanged from [7.0.2](release-notes/7.0/7.0.2.md), so upgrading from `7.0.2` to `7.1.0-preview3` does **not** require any new .NET Framework strong-name binding redirects. + +### Added + +- Added four `virtual` asynchronous counterparts to the synchronous methods on `SqlColumnEncryptionKeyStoreProvider` — `DecryptColumnEncryptionKeyAsync`, `EncryptColumnEncryptionKeyAsync`, `SignColumnMasterKeyMetadataAsync`, and `VerifyColumnMasterKeyMetadataAsync` — each accepting an optional `CancellationToken`. The default implementations delegate to the existing synchronous methods, so existing custom providers are unaffected. These APIs are introduced for provider authors but are not consumed by the driver yet; a future release will enable their use from the driver's own asynchronous APIs. + ([#3673](https://github.com/dotnet/SqlClient/pull/3673)) + +- Implemented those four asynchronous APIs in `SqlColumnEncryptionAzureKeyVaultProvider`, calling the Azure SDK's own asynchronous methods and flowing the supplied `CancellationToken`. Concurrent cache misses for the same key are gated so a burst of callers issues a single Key Vault request. See the [AKV release notes](release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0-preview3.md) for the `VerifyColumnMasterKeyMetadata` signature-validation behavior change and the 7.1 runtime requirement. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +- Substantially expanded `ChannelDbConnectionPool` (the opt-in pool behind `Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`) to parity with the default pool: transaction support, broken-connection replacement, leaked-connection reclamation, background warmup and replenishment to `Min Pool Size`, idle pruning driven by `Connection Idle Timeout`, optional connection-creation rate limiting, and metrics/tracing parity. Default pooling behavior is unchanged. + ([#4395](https://github.com/dotnet/SqlClient/pull/4395), + [#4396](https://github.com/dotnet/SqlClient/pull/4396), + [#4429](https://github.com/dotnet/SqlClient/pull/4429), + [#4452](https://github.com/dotnet/SqlClient/pull/4452), + [#4463](https://github.com/dotnet/SqlClient/pull/4463), + [#4487](https://github.com/dotnet/SqlClient/pull/4487), + [#4504](https://github.com/dotnet/SqlClient/pull/4504), + [#4529](https://github.com/dotnet/SqlClient/pull/4529)) + +### Changed + +- The driver now builds a single cross-platform assembly. Package structure and contents are unchanged, and Windows-only native SNI types now trim cleanly on Linux and macOS. + ([#4207](https://github.com/dotnet/SqlClient/pull/4207), + [#4465](https://github.com/dotnet/SqlClient/pull/4465), + [#4474](https://github.com/dotnet/SqlClient/pull/4474)) + +- Reduced managed allocations in the async read path by restoring reuse of `PacketData` nodes via a bounded free list on `StateSnapshot`. This applies to the default async read path and is not gated behind any AppContext switch. + ([#4536](https://github.com/dotnet/SqlClient/pull/4536)) + +- Operations no longer allocate a formatted trace string when `SqlClientEventSource` tracing is disabled, recovering a memory regression against the 6.1.6 baseline. Trace output with tracing enabled is unchanged. + ([#4528](https://github.com/dotnet/SqlClient/pull/4528)) + +- Updated centrally managed dependency versions for the `net9.0` target framework to `9.0.18`, and added `System.Threading.RateLimiting` to the packaged dependency metadata. Non-`net9.0` targets keep their existing `8.0.x` pins. + ([#4507](https://github.com/dotnet/SqlClient/pull/4507)) + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to `7.1.0-preview3.26226.3`. + ([#4564](https://github.com/dotnet/SqlClient/pull/4564)) + +- Bypassed SQL Graph column alias mapping in `SqlBulkCopy` when neither the source nor destination table contains graph pseudo-columns, recovering a bulk copy performance regression. Graph table bulk copy behavior is unchanged. + ([#4535](https://github.com/dotnet/SqlClient/pull/4535)) + +- Re-shipped `Microsoft.Data.SqlClient.Extensions.Azure` as `7.1.0-preview3`, fixing Entra ID tenant parsing for multi-segment authorities. See [release notes](release-notes/Extensions/Azure/7.1/7.1.0-preview3.md). + +- Re-shipped `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` as `7.1.0-preview3` with new asynchronous key store provider APIs, and `Microsoft.Data.SqlClient.Extensions.Abstractions` and `Microsoft.Data.SqlClient.Internal.Logging` as `7.1.0-preview3` (version alignment only, no functional changes). See release notes for [AKV](release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0-preview3.md), [Abstractions](release-notes/Extensions/Abstractions/7.1/7.1.0-preview3.md), and [Logging](release-notes/Internal/Logging/7.1/7.1.0-preview3.md). + +### Fixed + +- Fixed Always Encrypted VSM/HGS enclave attestation not verifying that the enclave public key used to establish the session matches the key committed to by the signed attestation report. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532)) + +- Fixed a `SqlConnectionFactory` timer that woke the process every 30 seconds for the lifetime of the application even when no connection pools existed, including with `Pooling=False` and after `ClearAllPools()`. + ([#4479](https://github.com/dotnet/SqlClient/pull/4479)) + +- Fixed connection pool performance counter defects affecting the default pool as well as pool V2. `active-soft-connects` and `number-of-active-connections` could go negative after a failed connection activation, and several gauges drifted upward permanently after a broken connection was replaced. + ([#4504](https://github.com/dotnet/SqlClient/pull/4504)) + +- Fixed `OverflowException` when sending large `decimal` values as a parameter with explicit `Precision` and `Scale`, which primarily affected Always Encrypted scenarios. + ([#4443](https://github.com/dotnet/SqlClient/pull/4443)) + +- Fixed a TDS stream error when passing a `DateOnly` value as a parameter with `SqlDbType.Variant`. + ([#4294](https://github.com/dotnet/SqlClient/pull/4294)) + +- Fixed `DateOnly` values written to a `sql_variant` column of a table-valued parameter being sent as `datetime` instead of `date`, which also caused an overflow for values out of `datetime` range. + ([#4439](https://github.com/dotnet/SqlClient/pull/4439)) + +- Fixed the `ServerCertificate` connection-string keyword not being honored when the platform reported no TLS policy errors, and made an unloadable certificate file fail closed instead of silently falling back to host-name validation. + ([#4445](https://github.com/dotnet/SqlClient/pull/4445)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, unlike `SqlConnection.AccessToken`, along with a related connection pool key defect. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520)) + +- Fixed several async entry points in `SqlBulkCopy`, `SqlDataReader`, and `SqlCommand` that captured fatal exceptions such as `OutOfMemoryException` into faulted `Task`s instead of letting them propagate. + ([#4437](https://github.com/dotnet/SqlClient/pull/4437)) + +- Fixed `Authentication=Active Directory Service Principal` (and the other Entra ID flows) failing against endpoints that return a multi-segment authority such as the Dataverse / Dynamics 365 TDS endpoint; the tenant id is now taken from the first path segment of the STSURL authority instead of the last. Ships in `Microsoft.Data.SqlClient.Extensions.Azure`. + ([#4521](https://github.com/dotnet/SqlClient/pull/4521)) + +## [Preview Release 7.1.0-preview2] - 2026-07-09 + +This update brings the following changes since the [7.1.0-preview1](release-notes/7.1/7.1.0-preview1.md) release. +See the [full release notes](release-notes/7.1/7.1.0-preview2.md) for detailed descriptions. + +> **Important — package version alignment:** Starting with the [7.0.2](release-notes/7.0/7.0.2.md) release, the `Microsoft.Data.SqlClient` driver and its companion packages share a single aligned version. Preview 2 of the `7.1` line continues this alignment; the following packages now ship together as `7.1.0-preview2`: +> +> - `Microsoft.Data.SqlClient` +> - `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` +> - `Microsoft.Data.SqlClient.Extensions.Azure` +> - `Microsoft.Data.SqlClient.Extensions.Abstractions` +> - `Microsoft.Data.SqlClient.Internal.Logging` +> +> (`Microsoft.SqlServer.Server` continues to version independently and remains at `1.0.0`.) +> +> Applications that reference `Microsoft.Data.SqlClient.Extensions.Azure` must upgrade it to `7.1.0-preview2` when upgrading `Microsoft.Data.SqlClient` to `7.1.0-preview2`. +> +> **Compatibility guarantee:** All aligned assemblies ship with `FileVersion 7.1.0.x` and `AssemblyVersion 7.0.0.0`. The `AssemblyVersion` is unchanged from [7.0.2](release-notes/7.0/7.0.2.md), so upgrading from `7.0.2` to `7.1.0-preview2` does **not** require any new .NET Framework strong-name binding redirects. See the 7.0.2 release notes for the original `AssemblyVersion` alignment (the one-time breaking change that raised `AssemblyVersion` from `1.0.0.0` to `7.0.0.0` for `Extensions.Azure`, `Extensions.Abstractions`, and `Internal.Logging`; `AzureKeyVaultProvider` was already on `7.x`). + +### Added + +- Added `SqlConnection.GetSchemaAsync` overloads mirroring the existing synchronous shapes. + ([#3005](https://github.com/dotnet/SqlClient/pull/3005)) + +- Added SQL Graph column-alias support (`$node_id`, `$edge_id`, `$from_id`, `$to_id`) as destination columns in `SqlBulkCopy`. + ([#3677](https://github.com/dotnet/SqlClient/pull/3677)) + +- `SqlBatchCommand.CommandBehavior` (a driver-specific property that existed since batching was introduced but was previously ignored) is now honored during `SqlBatch` execution, and `SqlBatch.ExecuteReader` now respects the `CommandBehavior` value passed to it. + ([#4125](https://github.com/dotnet/SqlClient/pull/4125)) + +- Added a `Connection Idle Timeout` connection-string keyword and matching `SqlConnectionStringBuilder.IdleTimeout` property to evict idle pooled connections (default `300` seconds; `0` disables). Enforcement is opt-in via `Switch.Microsoft.Data.SqlClient.UseLegacyIdleTimeoutBehavior=false`; the default preserves the historical pooling behavior. When enabled, idle-timeout enforcement applies to the existing connection pool as well. + ([#4295](https://github.com/dotnet/SqlClient/pull/4295)) + +### Changed + +- The `Connect Timeout` budget can now be propagated through pool acquisition via a shared `TimeoutTimer`, so time spent waiting in the pool is deducted from the overall timeout. Enforcement is opt-in via `Switch.Microsoft.Data.SqlClient.UseOverallConnectTimeoutForPoolWait=true`; the default (`false`) preserves the historical behavior where pool waits do not count against `Connect Timeout`. Adds a dependency on `Microsoft.Bcl.TimeProvider`. + ([#4270](https://github.com/dotnet/SqlClient/pull/4270)) + +- Hardened `SqlConnection` internal state transitions with `Interlocked.CompareExchange` guards. + ([#4267](https://github.com/dotnet/SqlClient/pull/4267)) + +- Removed legacy connection-options inheritance from internal APIs and simplified `TryOpenConnection` / `TryReplaceConnection` / pool interfaces. + ([#4235](https://github.com/dotnet/SqlClient/pull/4235), + [#4237](https://github.com/dotnet/SqlClient/pull/4237), + [#4261](https://github.com/dotnet/SqlClient/pull/4261), + [#4415](https://github.com/dotnet/SqlClient/pull/4415)) + +- Added the SQL Server 2025 `json` data type to the `DataTypes` schema table returned by `SqlConnection.GetSchema`. + ([#3858](https://github.com/dotnet/SqlClient/pull/3858)) + +- Added async generic helpers to reduce sync/async duplication. + ([#4334](https://github.com/dotnet/SqlClient/pull/4334)) + +- Use hardcoded LCID mappings when decoding strings. + ([#4212](https://github.com/dotnet/SqlClient/pull/4212)) + +- Reduced allocations by skipping lock acquisition on `SqlErrorCollection` counters when no errors exist, and by avoiding stack-trace materialization on expected `null`-return paths. + ([#4099](https://github.com/dotnet/SqlClient/pull/4099), + [#4102](https://github.com/dotnet/SqlClient/pull/4102), + [#4157](https://github.com/dotnet/SqlClient/pull/4157)) + +- Improved accuracy of `EnclaveDiffieHellmanInfo.Size`. + ([#4346](https://github.com/dotnet/SqlClient/pull/4346)) + +- `SqlVector` now serializes and deserializes multibyte values as little-endian explicitly. + ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) + +- Updated the bundled .NET 10 SDK to `10.0.300`. + ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) + +- Re-shipped `Microsoft.Data.SqlClient.Extensions.Azure` as `7.1.0-preview2`, adding Windows Account Manager (WAM) broker support for Entra ID authentication on Windows. See [release notes](release-notes/Extensions/Azure/7.1/7.1.0-preview2.md). + +- Re-shipped `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` as `7.1.0-preview2` (version alignment only, no functional changes). See release notes for [AKV](release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0-preview2.md), [Abstractions](release-notes/Extensions/Abstractions/7.1/7.1.0-preview2.md), and [Logging](release-notes/Internal/Logging/7.1/7.1.0-preview2.md). + +### Fixed + +- Fixed a `NullReferenceException` in `SqlCommand.Cancel()` when the active connection has already been torn down. + ([#4372](https://github.com/dotnet/SqlClient/pull/4372)) + +- Fixed Always Encrypted column master key signature verification incorrectly reusing cached results after a prior verification failure. + ([#4339](https://github.com/dotnet/SqlClient/pull/4339)) + +- Fixed missing bounds checks on TDS token and feature-extension-acknowledgment data lengths that could allow a spoofing server to trigger unbounded allocations. + ([#4340](https://github.com/dotnet/SqlClient/pull/4340)) + +- Fixed `SqlBulkCopy` failing in least-privilege environments. + ([#4306](https://github.com/dotnet/SqlClient/pull/4306)) + +- Fixed Always Encrypted reads of `CekMdVersion` and `EkValueCount` to align with the TDS specification. + ([#4240](https://github.com/dotnet/SqlClient/pull/4240)) + +- Fixed `LoginWithFailover` to validate parser state before continuing. + ([#4140](https://github.com/dotnet/SqlClient/pull/4140)) + +- Fixed the SPN used during login to use the resolved port instead of the instance name when `Protocol=None` or `Protocol=Admin`. + ([#4180](https://github.com/dotnet/SqlClient/pull/4180)) + +- Fixed a race in `SqlConnection.TryOpenInner` that could surface as `InvalidCastException`; the same race now returns a deterministic `InvalidOperationException`. + ([#4179](https://github.com/dotnet/SqlClient/pull/4179)) + +- Fixed several `CancellationTokenSource` leaks across `SqlDataReader`, `SqlConnection` reconnect, `SqlCommand` reconnect timeout, and sequential-stream helpers. + ([#4009](https://github.com/dotnet/SqlClient/pull/4009)) + +- Fixed server certificate documentation. + ([#4408](https://github.com/dotnet/SqlClient/pull/4408)) + +### Removed + +- **Breaking:** Removed SQL Server 7.0 and SQL Server 2000 support, along with the `TypeSystem.SQLServer2000` enum value and the `Type System Version=SQL Server 2000` connection-string value. Connection strings that specify this value now throw `ArgumentException` when the connection is opened. Supported values are `Latest`, `SQL Server 2005`, `SQL Server 2008`, and `SQL Server 2012`. + ([#4015](https://github.com/dotnet/SqlClient/pull/4015)) + +## [Stable Release 7.0.2] - 2026-06-24 + +This update brings the following changes since the [7.0.1](release-notes/7.0/7.0.1.md) release. +See the [full release notes](release-notes/7.0/7.0.2.md) for detailed descriptions. + +> **Important — package version alignment:** Starting with 7.0.2, the `Microsoft.Data.SqlClient` driver and its companion packages share a single aligned version. The following packages now ship together as `7.0.2`: +> +> - `Microsoft.Data.SqlClient` +> - `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` +> - `Microsoft.Data.SqlClient.Extensions.Azure` +> - `Microsoft.Data.SqlClient.Extensions.Abstractions` +> - `Microsoft.Data.SqlClient.Internal.Logging` +> +> (`Microsoft.SqlServer.Server` continues to version independently and remains at `1.0.0`.) +> +> Applications must reference the same versions of `Microsoft.Data.SqlClient` and its extensions for best compatibility. In particular, applications that reference `Microsoft.Data.SqlClient.Extensions.Azure` must upgrade it to `7.0.2` when upgrading `Microsoft.Data.SqlClient` to `7.0.2`. + +> **Breaking change (.NET Framework only):** As part of this alignment, the `AssemblyVersion` of `Microsoft.Data.SqlClient.Extensions.Azure`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` changed from `1.0.0.0` to `7.0.0.0`. On .NET Framework, `AssemblyVersion` is part of the strong-name identity, so applications that drop these assemblies into an existing deployment without rebuilding must rebuild against the 7.0.2 packages (or add binding redirects). Applications on .NET / .NET Core are not affected. `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` already used a `7.x` assembly version and is unaffected. + +### Fixed + +- Fixed a `NullReferenceException` in `SqlCommand.Cancel()` when the active connection has already been torn down. + ([#4372](https://github.com/dotnet/SqlClient/pull/4372), + [#4373](https://github.com/dotnet/SqlClient/pull/4373)) + +- Fixed a `NullReferenceException` in `SqlDataReader.GetBytes`/`GetChars` when called with a `null` destination buffer. + ([#4159](https://github.com/dotnet/SqlClient/pull/4159), + [#4206](https://github.com/dotnet/SqlClient/pull/4206)) + +- Fixed Always Encrypted column master key signature verification incorrectly reusing cached results. + ([#4339](https://github.com/dotnet/SqlClient/pull/4339), + [#4343](https://github.com/dotnet/SqlClient/pull/4343)) + +### Changed + +- Hardened TDS token parsing by adding data-length bounds checks for token and feature-extension-acknowledgment data. + ([#4340](https://github.com/dotnet/SqlClient/pull/4340), + [#4358](https://github.com/dotnet/SqlClient/pull/4358)) + +- Released `Microsoft.Data.SqlClient.Extensions.Azure 7.0.2`, adding WAM broker support for Entra ID authentication modes on Windows. See [release notes](release-notes/Extensions/Azure/7.0/7.0.2.md). + ([#4288](https://github.com/dotnet/SqlClient/pull/4288), + [#4388](https://github.com/dotnet/SqlClient/pull/4388)) + +- Re-shipped `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` as `7.0.2` (version alignment only, no functional changes). See release notes for [AKV](release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.2.md), [Abstractions](release-notes/Extensions/Abstractions/7.0/7.0.2.md), and [Logging](release-notes/Internal/Logging/7.0/7.0.2.md). + +## [Stable Release 6.1.6] - 2026-06-24 + +This update brings the following changes since the [6.1.5](release-notes/6.1/6.1.5.md) release. +See the [full release notes](release-notes/6.1/6.1.6.md) for detailed descriptions. + +### Added + +- Added Web Account Manager (WAM) broker support for the supported Entra ID authentication modes (Windows only), including the new `ActiveDirectoryAuthenticationProviderOptions` type with a `UseWamBroker` property, an `ActiveDirectoryAuthenticationProvider(ActiveDirectoryAuthenticationProviderOptions options)` constructor overload, and a cross-platform `SetParentActivityOrWindowFunc(Func)` method. + ([#4288](https://github.com/dotnet/SqlClient/pull/4288), + [#4387](https://github.com/dotnet/SqlClient/pull/4387)) + +### Changed + +- Hardened TDS token parsing with data-length bounds checks to prevent unbounded memory allocation from a server spoofing length fields. + ([#4340](https://github.com/dotnet/SqlClient/pull/4340), + [#4359](https://github.com/dotnet/SqlClient/pull/4359)) + +- Updated dependencies ([#4387](https://github.com/dotnet/SqlClient/pull/4387)): + - Updated `Microsoft.Identity.Client` to 4.84.2 (was 4.80.0) + - Added `Microsoft.Identity.Client.Broker` 4.84.2 + +### Fixed + +- Fixed a `NullReferenceException` in `SqlDataReader.GetChars` on the PLP + `SequentialAccess` path when a `null` buffer was passed with a negative `bufferIndex`; it now throws `ArgumentOutOfRangeException`. + ([#4159](https://github.com/dotnet/SqlClient/pull/4159), + [#4205](https://github.com/dotnet/SqlClient/pull/4205)) + +- Fixed column master key (CMK) signature verification caching where a cached verification failure could subsequently be reported as a valid signature. + ([#4339](https://github.com/dotnet/SqlClient/pull/4339), + [#4356](https://github.com/dotnet/SqlClient/pull/4356)) + +## [Preview Release 7.1.0-preview1] - 2026-04-29 + +This update brings the following changes since the [7.0.0](release-notes/7.0/7.0.0.md) release. +See the [full release notes](release-notes/7.1/7.1.0-preview1.md) for detailed descriptions. + +### Added + +- Added `SqlBatch` support on .NET Framework so the batching API is available across the full supported platform matrix. + ([#3926](https://github.com/dotnet/SqlClient/pull/3926)) + +- Added additional accepted connection-string synonyms for cross-driver alignment, including `ColumnEncryption`, `ConnectTimeout`, `FailoverPartner`, `PacketSize`, and `WorkstationId`. + ([#4192](https://github.com/dotnet/SqlClient/pull/4192)) + +### Changed + +- `SqlConnection.ClearPool` and `SqlConnection.ClearAllPools` now work correctly with the channel-based pool implementation. + ([#4194](https://github.com/dotnet/SqlClient/pull/4194)) + +- Added type forwards for public authentication abstractions moved into `Microsoft.Data.SqlClient.Extensions.Abstractions`. + ([#4067](https://github.com/dotnet/SqlClient/pull/4067), + [#4117](https://github.com/dotnet/SqlClient/pull/4117)) + +- Enabled the User Agent feature extension by default. + ([#4124](https://github.com/dotnet/SqlClient/pull/4124), + [#4154](https://github.com/dotnet/SqlClient/pull/4154)) + +- Reduced allocations when sending large string values, plus related source-build and packaging consolidation updates. + ([#4072](https://github.com/dotnet/SqlClient/pull/4072), + [#4033](https://github.com/dotnet/SqlClient/pull/4033), + [#4068](https://github.com/dotnet/SqlClient/pull/4068), + [#4204](https://github.com/dotnet/SqlClient/pull/4204)) + +### Fixed + +- Fixed `SqlBulkCopy` on SQL Server 2016 graph tables by extracting graph metadata with dynamic SQL. + ([#3714](https://github.com/dotnet/SqlClient/issues/3714), + [#4092](https://github.com/dotnet/SqlClient/pull/4092), + [#4147](https://github.com/dotnet/SqlClient/pull/4147)) + +- Fixed `SqlBulkCopy` support for Azure Synapse Analytics dedicated SQL pools. + ([#4149](https://github.com/dotnet/SqlClient/issues/4149), + [#4176](https://github.com/dotnet/SqlClient/pull/4176), + [#4182](https://github.com/dotnet/SqlClient/pull/4182)) + +- Fixed vector float32 metadata so `GetFieldType()` and `GetProviderSpecificFieldType()` return the expected vector type. + ([#4104](https://github.com/dotnet/SqlClient/issues/4104), + [#4105](https://github.com/dotnet/SqlClient/pull/4105), + [#4152](https://github.com/dotnet/SqlClient/pull/4152)) + +- Added the missing `System.Data.Common` dependency for .NET Framework consumers. + ([#4063](https://github.com/dotnet/SqlClient/pull/4063), + [#4074](https://github.com/dotnet/SqlClient/pull/4074)) + +- Fixed a `SqlDataReader` streaming bug triggered by calling `IsDBNull()` before reading streamed column data. + ([#4082](https://github.com/dotnet/SqlClient/pull/4082)) + +- Fixed a `NullReferenceException` in `SqlDataReader`. + ([#4159](https://github.com/dotnet/SqlClient/pull/4159)) + +## [Stable Release 6.1.5] - 2026-04-27 + +This update brings the following changes since the [6.1.4](release-notes/6.1/6.1.4.md) release. +See the [full release notes](release-notes/6.1/6.1.5.md) for target platform support and dependency information. + +### Fixed + +- Fixed a connection performance regression where SPN (Service Principal Name) generation was triggered for non-integrated authentication modes (e.g., SQL authentication) on the native SNI path, causing unnecessary DNS lookups and significantly slower connection times. + ([#3523](https://github.com/dotnet/SqlClient/issues/3523), [#3946](https://github.com/dotnet/SqlClient/pull/3946)) + +- Fixed `ExecuteScalar` to properly propagate errors when the server sends data followed by an error token. Previously, errors such as conversion failures during `WHERE` clause evaluation were silently consumed during `SqlDataReader.Close()` instead of being thrown to the caller, which could result in transactions being unexpectedly zombied. + ([#3736](https://github.com/dotnet/SqlClient/issues/3736), [#3947](https://github.com/dotnet/SqlClient/pull/3947)) + +- Fixed `SqlDataReader.GetFieldType` and `SqlDataReader.GetProviderSpecificFieldType` to return the correct type (`SqlVector`) for vector float32 columns. + ([#4104](https://github.com/dotnet/SqlClient/issues/4104), [#4151](https://github.com/dotnet/SqlClient/pull/4151)) + +## [Stable Release 7.0.1] - 2026-04-23 + +This update brings the following changes since the [7.0.0](release-notes/7.0/7.0.0.md) release. +See the [full release notes](release-notes/7.0/7.0.1.md) for detailed descriptions. + +### Fixed + +- Fixed `SqlBulkCopy` failing on SQL Server 2016 with `Invalid column name 'graph_type'` error by using dynamic SQL to extract column names. + ([#3714](https://github.com/dotnet/SqlClient/issues/3714), + [#4092](https://github.com/dotnet/SqlClient/pull/4092), + [#4147](https://github.com/dotnet/SqlClient/pull/4147)) + +- Fixed `SqlBulkCopy` failing on Azure Synapse Analytics dedicated SQL pools by using `STRING_AGG` for the column-list query when targeting Synapse. + ([#4149](https://github.com/dotnet/SqlClient/issues/4149), + [#4176](https://github.com/dotnet/SqlClient/pull/4176), + [#4182](https://github.com/dotnet/SqlClient/pull/4182)) + +- Fixed `SqlDataReader.GetFieldType()` and `GetProviderSpecificFieldType()` returning incorrect type for vector float32 columns. + ([#4104](https://github.com/dotnet/SqlClient/issues/4104), + [#4105](https://github.com/dotnet/SqlClient/pull/4105), + [#4152](https://github.com/dotnet/SqlClient/pull/4152)) + +- Added missing `System.Data.Common` (v4.3.0) NuGet package dependency for .NET Framework consumers to resolve `CS0012` compilation errors. + ([#4063](https://github.com/dotnet/SqlClient/pull/4063), + [#4074](https://github.com/dotnet/SqlClient/pull/4074)) + +### Changed + +- Enabled the User Agent TDS feature extension unconditionally; removed the `Switch.Microsoft.Data.SqlClient.EnableUserAgent` AppContext switch. + ([#4124](https://github.com/dotnet/SqlClient/pull/4124), + [#4154](https://github.com/dotnet/SqlClient/pull/4154)) + +- Added type forwards from the core assembly to public types moved to `Microsoft.Data.SqlClient.Extensions.Abstractions`. + ([#4067](https://github.com/dotnet/SqlClient/pull/4067), + [#4117](https://github.com/dotnet/SqlClient/pull/4117)) + +- Fixed API documentation include paths and duplicate doc snippets. + ([#4084](https://github.com/dotnet/SqlClient/pull/4084), + [#4086](https://github.com/dotnet/SqlClient/pull/4086), + [#4107](https://github.com/dotnet/SqlClient/pull/4107), + [#4161](https://github.com/dotnet/SqlClient/pull/4161)) + +## [Stable Release 7.0.0] - 2026-03-17 + +This section summarizes all changes across the 7.0 preview cycle for users upgrading from the latest 6.1 stable release. +See the [full release notes](release-notes/7.0/7.0.0.md) for detailed descriptions. + +Also released as part of this milestone: +- Released Microsoft.Data.SqlClient.Extensions.Abstractions 1.0.0. See [release notes](release-notes/Extensions/Abstractions/1.0/1.0.0.md). +- Released Microsoft.Data.SqlClient.Extensions.Azure 1.0.0. See [release notes](release-notes/Extensions/Azure/1.0/1.0.0.md). +- Released Microsoft.Data.SqlClient.Internal.Logging 1.0.0. See [release notes](release-notes/Internal/Logging/1.0/1.0.0.md). +- Released Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.0.0. See [release notes](release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.0.md). + +### Changed + +- **Breaking:** Removed Azure dependencies from the core package. Entra ID authentication (`ActiveDirectoryAuthenticationProvider` and related types) has been extracted into a new `Microsoft.Data.SqlClient.Extensions.Azure` package. The core `Microsoft.Data.SqlClient` package no longer depends on `Azure.Core`, `Azure.Identity`, or their transitive dependencies. Applications using Entra ID authentication must now install `Microsoft.Data.SqlClient.Extensions.Azure` separately. + ([#1108](https://github.com/dotnet/SqlClient/issues/1108), + [#3680](https://github.com/dotnet/SqlClient/pull/3680), + [#3902](https://github.com/dotnet/SqlClient/pull/3902), + [#3904](https://github.com/dotnet/SqlClient/pull/3904), + [#3908](https://github.com/dotnet/SqlClient/pull/3908), + [#3917](https://github.com/dotnet/SqlClient/pull/3917), + [#3982](https://github.com/dotnet/SqlClient/pull/3982), + [#3978](https://github.com/dotnet/SqlClient/pull/3978), + [#3986](https://github.com/dotnet/SqlClient/pull/3986)) + +- Two additional packages were introduced to support this separation: `Microsoft.Data.SqlClient.Extensions.Abstractions` (shared types between the core driver and extensions) and `Microsoft.Data.SqlClient.Internal.Logging` (shared ETW tracing infrastructure). + ([#3626](https://github.com/dotnet/SqlClient/pull/3626), + [#3628](https://github.com/dotnet/SqlClient/pull/3628), + [#3967](https://github.com/dotnet/SqlClient/pull/3967), + [#4038](https://github.com/dotnet/SqlClient/pull/4038)) + +- Deprecated `SqlAuthenticationMethod.ActiveDirectoryPassword` (ROPC flow). The method is now marked `[Obsolete]` and will generate compiler warnings. Migrate to `ActiveDirectoryInteractive`, `ActiveDirectoryServicePrincipal`, `ActiveDirectoryManagedIdentity`, or `ActiveDirectoryDefault`. + ([#3671](https://github.com/dotnet/SqlClient/pull/3671)) + +- Reverted public visibility of internal interop enums (`IoControlCodeAccess` and `IoControlTransferType`) that were accidentally made public during the project merge. + ([#3900](https://github.com/dotnet/SqlClient/pull/3900)) + +- Removed `Constrained Execution Region` error handling blocks and associated `SqlConnection` cleanup. + ([#3535](https://github.com/dotnet/SqlClient/pull/3535)) + +- Performance improvements across SqlStatistics timing, Always Encrypted scenarios, and connection opening: + ([#3609](https://github.com/dotnet/SqlClient/pull/3609), + [#3612](https://github.com/dotnet/SqlClient/pull/3612), + [#3732](https://github.com/dotnet/SqlClient/pull/3732), + [#3660](https://github.com/dotnet/SqlClient/pull/3660), + [#3791](https://github.com/dotnet/SqlClient/pull/3791), + [#3772](https://github.com/dotnet/SqlClient/pull/3772), + [#3554](https://github.com/dotnet/SqlClient/pull/3554)) + +- Allow `SqlBulkCopy` to operate on hidden columns. + ([#3590](https://github.com/dotnet/SqlClient/pull/3590)) + +- Updated UserAgent feature to use a pipe-delimited format, replacing the previous JSON format. + ([#3826](https://github.com/dotnet/SqlClient/pull/3826)) + +- Minor improvements to Managed SNI tracing to capture continuation events and errors. + ([#3859](https://github.com/dotnet/SqlClient/pull/3859)) + +### Added + +- Added `SspiContextProvider` abstract class and `SqlConnection.SspiContextProvider` property, enabling custom SSPI authentication for scenarios like cross-domain Kerberos negotiation and NTLM username/password authentication. + ([#2253](https://github.com/dotnet/SqlClient/issues/2253), + [#2494](https://github.com/dotnet/SqlClient/pull/2494)) + +- Continued refinement of packet multiplexing with bug fixes and stability improvements, plus new app context switches for opt-in control. + ([#3534](https://github.com/dotnet/SqlClient/pull/3534), + [#3537](https://github.com/dotnet/SqlClient/pull/3537), + [#3605](https://github.com/dotnet/SqlClient/pull/3605)) + +- Added support for enhanced routing, a TDS feature that allows the server to redirect connections to a specific server and database during login, enabling Azure SQL Hyperscale read replica load balancing. + ([#3641](https://github.com/dotnet/SqlClient/issues/3641), + [#3969](https://github.com/dotnet/SqlClient/pull/3969), + [#3970](https://github.com/dotnet/SqlClient/pull/3970), + [#3973](https://github.com/dotnet/SqlClient/pull/3973)) + +- Updated pipelines and test suites to compile the driver using the .NET 10 SDK. + ([#3686](https://github.com/dotnet/SqlClient/pull/3686)) + +- Added `SqlConfigurableRetryFactory.BaselineTransientErrors` static property exposing the default transient error codes list as a `ReadOnlyCollection`. + ([#3903](https://github.com/dotnet/SqlClient/pull/3903)) + +- Added app context switch `Switch.Microsoft.Data.SqlClient.EnableMultiSubnetFailoverByDefault` to set `MultiSubnetFailover=true` globally without modifying connection strings. + ([#3841](https://github.com/dotnet/SqlClient/pull/3841)) + +- Added app context switch `Switch.Microsoft.Data.SqlClient.IgnoreServerProvidedFailoverPartner` to let the client ignore server-provided failover partner info in Basic Availability Groups. + ([#3625](https://github.com/dotnet/SqlClient/pull/3625)) + +- Enabled `SqlClientDiagnosticListener` for `SqlCommand` on .NET Framework, closing a long-standing observability gap where diagnostic events were previously only available on .NET Core. + ([#3658](https://github.com/dotnet/SqlClient/pull/3658)) + +- Brought the 15 strongly-typed diagnostic event classes in the `Microsoft.Data.SqlClient.Diagnostics` namespace (e.g., `SqlClientCommandBefore`, `SqlClientConnectionOpenAfter`, `SqlClientTransactionCommitError`) to .NET Framework as part of the codebase merge. These types were originally introduced for .NET Core in 6.0. + ([#3493](https://github.com/dotnet/SqlClient/pull/3493)) + +- Enabled User Agent Feature Extension (opt-in via `Switch.Microsoft.Data.SqlClient.EnableUserAgent`). + ([#3606](https://github.com/dotnet/SqlClient/pull/3606)) + +- Added actionable error message when Entra ID authentication methods are used without the `Microsoft.Data.SqlClient.Extensions.Azure` package installed. + ([#3962](https://github.com/dotnet/SqlClient/issues/3962), + [#4046](https://github.com/dotnet/SqlClient/pull/4046)) + +### Fixed + +- Fixed a connection performance regression where SPN generation was triggered for non-integrated authentication modes (e.g., SQL authentication) on the native SNI path. + ([#3929](https://github.com/dotnet/SqlClient/pull/3929)) + +- Fixed `ExecuteScalar` to propagate errors when the server sends data followed by an error token. + ([#3912](https://github.com/dotnet/SqlClient/pull/3912)) + +- Fixed `NullReferenceException` in `SqlDataAdapter` when processing batch scenarios. + ([#3857](https://github.com/dotnet/SqlClient/pull/3857)) + +- Fixed reading of multiple app context switches from a single `AppContextSwitchOverrides` configuration field. + ([#3960](https://github.com/dotnet/SqlClient/pull/3960)) + +- Fixed an edge case in `TdsParserStateObject.TryReadPlpBytes` where zero-length reads returned `null` instead of an empty array. + ([#3872](https://github.com/dotnet/SqlClient/pull/3872)) + +- Fixed issue where extra connection deactivation was occurring. + ([#3758](https://github.com/dotnet/SqlClient/pull/3758)) + +- Fixed debug assertion in connection pool (no impact to production code). + ([#3587](https://github.com/dotnet/SqlClient/pull/3587)) + +- Prevented uninitialized performance counters escaping `CreatePerformanceCounters`. + ([#3623](https://github.com/dotnet/SqlClient/pull/3623)) + +- Fixed `SetProvider` to return immediately if user-defined authentication provider found. + ([#3620](https://github.com/dotnet/SqlClient/pull/3620)) + +- Fixed connection pool concurrency issue. + ([#3632](https://github.com/dotnet/SqlClient/pull/3632)) + ## [Preview Release 7.0.0-preview4.26064.3] - 2026-03-05 This update brings the below changes over the previous preview release: @@ -307,7 +895,7 @@ This update brings the following changes over the previous preview release: *What Changed:* -- Updated pipelines and test suites to compile the driver using the .NET 10 SDK. Cleaned up unnecessary dependency references. +- Updated pipelines and test suites to compile the driver using the .NET 10 SDK. Cleaned up unnecessary dependency references. ([#3686](https://github.com/dotnet/SqlClient/pull/3686)) *Who Benefits:* @@ -514,7 +1102,7 @@ This update brings the following changes since [7.0.0-preview1.25257.1] #### Other changes -- Improve performance in `SqlStatistics` by using `Environment.TickCount` for calculating execution timing +- Improve performance in `SqlStatistics` by using `Environment.TickCount` for calculating execution timing ([#3609](https://github.com/dotnet/SqlClient/pull/3609)) - Improve performance in Always Encrypted scenarios by using lower-allocation primitives @@ -1186,7 +1774,7 @@ This update brings the below changes over the previous release: - Added support for Georgian collation [#2194](https://github.com/dotnet/SqlClient/pull/2194) - Added Localization support on .NET [#2210](https://github.com/dotnet/SqlClient/pull/2110) - Added .NET 8 support [#2230](https://github.com/dotnet/SqlClient/pull/2230) -- Added explicit version for major .NET version dependencies on System.Runtime.Caching 8.0.0, System.Configuration.ConfigurationManager 8.0.0, and System.Diagnostics. +- Added explicit version for major .NET version dependencies on System.Runtime.Caching 8.0.0, System.Configuration.ConfigurationManager 8.0.0, and System.Diagnostics. - DiagnosticSource 8.0.0 [#2303](https://github.com/dotnet/SqlClient/pull/2303) ### Fixed @@ -1424,7 +2012,7 @@ This update brings the below changes over the previous release: - Moved to new System.Data.SqlTypes APIs in **.NET 7** and upper. [1934](https://github.com/dotnet/SqlClient/pull/1934) and [#1981](https://github.com/dotnet/SqlClient/pull/1981) - Changed **[UseOneSecFloorInTimeoutCalculationDuringLogin](https://learn.microsoft.com/sql/connect/ado-net/appcontext-switches#enable-a-minimum-timeout-during-login)** App Context switch default to **true** and extended its effect to .NET and .NET Standard. [#2012](https://github.com/dotnet/SqlClient/pull/2012) -- Updated `Microsoft.Identity.Client` version from 4.47.2 to 4.53.0. [#2031](https://github.com/dotnet/SqlClient/pull/2031), [#2055](https://github.com/dotnet/SqlClient/pull/2055) +- Updated `Microsoft.Identity.Client` version from 4.47.2 to 4.53.0. [#2031](https://github.com/dotnet/SqlClient/pull/2031), [#2055](https://github.com/dotnet/SqlClient/pull/2055) - Code health improvement: [#1985](https://github.com/dotnet/SqlClient/pull/1985) ## [Stable Release 2.1.6] - 2023-04-27 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8dca7be35c..df139c5482 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,9 +129,49 @@ To prevent closure, simply add comments, push commits, or respond to feedback. Security issues and bugs should be reported privately, via email, to the Microsoft Security Response Center (MSRC) [secure@microsoft.com](mailto:secure@microsoft.com). You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Further information, including the MSRC PGP key, can be found in the [MSRC FAQ](https://www.microsoft.com/en-us/msrc/faqs-report-an-issue?rtc=1&oneroute=true). +## Submitting Pull Requests + +- **New features from community PRs must be driven by creating a GitHub issue first.** Discuss the proposal in the issue before starting implementation. This helps avoid wasted effort and ensures alignment with project goals. +- **Community contributions must align with project priorities.** We prioritize features and fixes according to our [published milestones](https://github.com/dotnet/SqlClient/milestones). PRs that conflict with or distract from active work may be deferred. +- **Our maintainers reserve the right to reject PRs** that do not meet the required criteria to qualify for review. This includes PRs that: + - Lack a corresponding approved issue (i.e., an issue that has been reviewed, acknowledged, and agreed upon by maintainers — typically indicated by the **`PM Approved`** field being set to **`Approved`** in the GitHub Project board) + - Introduce breaking changes without prior discussion + - Do not follow the project's coding standards and conventions + - Are missing adequate test coverage + - Conflict with work already in progress by the team +- **Bug fixes and small improvements** are generally welcome without a prior issue, but a linked issue helps us triage and prioritize your contribution. + +### What Makes a Great Contribution + +1. **Start with an issue** — File a [feature request](https://github.com/dotnet/SqlClient/issues/new?template=feature_request.md) or [bug report](https://github.com/dotnet/SqlClient/issues/new?template=bug-report.md) and wait for maintainer feedback +2. **Follow the conventions** — See [coding guidelines](policy/coding-style.md) +3. **Include tests** — Both unit tests and integration tests where applicable +4. **Keep scope focused** — One feature or fix per PR +5. **Update documentation** — For any public API changes + +### Contribution Workflow + +See [contributing-workflow.md](contributing-workflow.md) for details on how PRs are tracked through our review process. + +## Feedback + +The best way to give feedback is to create issues in the [dotnet/SqlClient](https://github.com/dotnet/SqlClient) repo. + +Please give us feedback that will provide insight on the following: + +- Existing features that are missing some capability or otherwise don't work well enough. +- Missing features that should be added to the product. +- Design choices for a feature that is currently in-progress. + +Some important caveats: + +- It is best to give design feedback quickly for improvements that are in development. We're unlikely to hold a feature from a release on late feedback. +- We are most likely to include improvements that either have a positive impact on a broad scenario or have very significant positive impact on a niche scenario. This means that we are unlikely to prioritize modest improvements to niche scenarios. +- Compatibility will almost always be given a higher priority than improvements. + ## Contribution Standards -Project maintainers will merge changes that improve the product significantly and broadly and that align with the [Microsoft.Data.SqlClient roadmap](roadmap.md). +Project maintainers will merge changes that improve the product significantly and broadly and that align with the [Microsoft.Data.SqlClient roadmap](https://github.com/dotnet/SqlClient/wiki/Roadmap). ### Requirements diff --git a/Directory.Packages.props b/Directory.Packages.props index d043543e78..ad71a41be4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,4 +1,33 @@ + + + + + + + + true - + + + + 7.1.0 + [$(SniVersion), $([MSBuild]::Add($(SniVersion.Split('.')[0]), 1)).0.0) + - + + $([MSBuild]::Add($(SqlServerPackageVersion.Trim().Split('.')[0]), 1)).0.0 + $([MSBuild]::Add($(SqlClientPackageVersion.Trim().Split('.')[0]), 1)).0.0 + - - - - + + + + Version="[$(SqlClientPackageVersion), $(SqlClientVersionCeiling))" /> - + Version="[$(SqlClientPackageVersion), $(SqlClientVersionCeiling))" /> + Version="[$(SqlClientPackageVersion), $(SqlClientVersionCeiling))" /> + + Version="[$(SqlClientPackageVersion), $(SqlClientVersionCeiling))" /> @@ -51,18 +97,35 @@ - - + - + + + + + + + + @@ -83,17 +146,19 @@ + - - + + + @@ -103,13 +168,17 @@ - - - + + + + + + + diff --git a/NuGet.analysis.config b/NuGet.analysis.config new file mode 100644 index 0000000000..1ba52a93d3 --- /dev/null +++ b/NuGet.analysis.config @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NuGet.config b/NuGet.config index a16ff70302..5bdc9722d5 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,19 +5,43 @@ + + + - + + + + + + + + + + + - - + + + + + + + + + + diff --git a/README.md b/README.md index 1a066c52d4..bf77802f15 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,9 @@ When targeting .NET on Windows, a package reference to [Microsoft.Data.SqlClient | Coding Style | [coding-style.md](/policy/coding-style.md) | | Contributing | [CONTRIBUTING.md](CONTRIBUTING.md) | | Copyright Information | [COPYRIGHT.md](COPYRIGHT.md) | +| Release Milestones | [GitHub milestones](https://github.com/dotnet/SqlClient/milestones) | | Review Process | [review-process.md](/policy/review-process.md) | +| Roadmap | [SqlClient roadmap](https://github.com/dotnet/SqlClient/wiki/Roadmap) | | Support Policy | [SUPPORT.md](SUPPORT.md) | ## Our Featured Contributors diff --git a/TESTGUIDE.md b/TESTGUIDE.md new file mode 100644 index 0000000000..206092b597 --- /dev/null +++ b/TESTGUIDE.md @@ -0,0 +1,328 @@ +# Test Guide for Microsoft.Data.SqlClient + +This guide describes how to run the test projects in this repository and how to configure the SQL Server-backed manual +tests. + +For build prerequisites and general `build.proj` usage, see [BUILDGUIDE.md](BUILDGUIDE.md). + +## Test Projects + +The primary test projects for Microsoft.Data.SqlClient are under +[src/Microsoft.Data.SqlClient/tests](src/Microsoft.Data.SqlClient/tests): + +| Project | Path | Purpose | +|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| +| Unit tests | [src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj](src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj) | Unit tests and tests against simulated servers. | +| Functional tests | [src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj](src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj) | Functional tests for public and internal behavior. Some tests use simulated servers or local test infrastructure. | +| Manual tests | [src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj](src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj) | Integration tests that generally require a configured SQL Server or Azure SQL target. | + +These projects target `net8.0`, `net9.0`, and `net10.0` on all platforms. On Windows, they also target `net462`. + +## Recommended Entry Point + +Use [build.proj](build.proj) from the repository root: + +```bash +dotnet build build.proj -t: [optional_parameters] +``` + +Since `build.proj` is the only project file in the repo root, it can be omitted when building from +the root: + +```bash +dotnet build -t: [optional_parameters] +``` + +The command-line examples below will assume that `build.proj` is selected by default and will omit +it from the `dotnet build` command. + +Test targets build the projects they depend on, so a separate build step is not required for normal test runs. + +## Test Targets + +| Target | Description | +|---------------------------|--------------------------------------------------------------------------------------------------------------------------| +| `Test` | Runs all test targets in the repository. This can take a long time and is not recommended for routine local development. | +| `TestAbstractions` | Runs Microsoft.Data.SqlClient.Extensions.Abstractions tests. | +| `TestAzure` | Runs Microsoft.Data.SqlClient.Extensions.Azure tests. | +| `TestSqlClient` | Runs all Microsoft.Data.SqlClient test projects. | +| `TestSqlClientUnit` | Runs Microsoft.Data.SqlClient unit tests. | +| `TestSqlClientFunctional` | Runs Microsoft.Data.SqlClient functional tests. | +| `TestSqlClientManual` | Runs Microsoft.Data.SqlClient manual tests. | + +## Common Commands + +Run the SqlClient unit tests: + +```bash +dotnet build -t:TestSqlClientUnit +``` + +Run the SqlClient functional tests: + +```bash +dotnet build -t:TestSqlClientFunctional +``` + +Run the SqlClient manual tests: + +```bash +dotnet build -t:TestSqlClientManual +``` + +Run only manual test set 2: + +```bash +dotnet build -t:TestSqlClientManual -p:TestSet=2 +``` + +Run manual test sets 1 and 3: + +```bash +dotnet build -t:TestSqlClientManual -p:TestSet=13 +``` + +Run Always Encrypted manual tests: + +```bash +dotnet build -t:TestSqlClientManual -p:TestSet=AE +``` + +Run a specific target framework: + +```bash +dotnet build -t:TestSqlClientFunctional -p:TestFramework=net8.0 +``` + +Run functional tests against an x86 `dotnet` installation: + +```bash +dotnet build -t:TestSqlClientFunctional -p:DotnetPath='C:\path\to\dotnet\x86\' +``` + +Run all Azure extension tests, including `interactive` tests, while still excluding tests marked `failing` or `flaky`: + +```bash +dotnet build -t:TestAzure -p:TestFilters=category!=failing +``` + +## Test Parameters + +The most commonly used test parameters are: + +| Parameter | Default | Description | +|-----------------------------|-----------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| +| `-p:Configuration=` | `Debug` | Build configuration. Use `Debug` or `Release`. | +| `-p:DotnetPath=` | Empty | Path to the folder containing the `dotnet` binary. The path must end with `\` or `/`. | +| `-p:ReferenceType=` | `Project` | For functional and manual SqlClient tests, use `Project` to test the source project or `Package` to test a package reference. | +| `-p:TestBlameTimeout=` | `10m` | Enables hang blame collection with the specified timeout. Use `0` to disable hang timeouts. | +| `-p:TestCodeCoverage=` | `true` | Collects code coverage when set to `true`. | +| `-p:TestFilters=` | `category!=failing&category!=flaky&category!=interactive` | xUnit filter expression. Use `none` to run without the default filter. | +| `-p:TestFramework=` | Empty | Target framework to run. If omitted, all target frameworks supported by the project and host OS are run. | +| `-p:TestResultsFolderPath=` | `test_results` | Directory where test results are written. | +| `-p:TestSet=` | Empty | Selects manual test sets. Supported values include `1`, `2`, `3`, `AE`, and combinations such as `13` or `12AE`. | + +## Test Filters + +`build.proj` passes `TestFilters` to `dotnet test --filter`. By default, tests marked with these categories are excluded: + +| Category | Why it is excluded by default | +|---------------|-------------------------------------------------------------------------------------| +| `failing` | Known failing tests. | +| `flaky` | Intermittently failing tests. | +| `interactive` | Tests that require user interaction or external setup not suitable for normal runs. | + +Examples: + +Run a single test by fully-qualified name: + +```bash +dotnet build -t:TestSqlClientUnit -p:TestFilters=FullyQualifiedName=Namespace.ClassName.MethodName +``` + +Run only flaky tests while investigating quarantine failures: + +```bash +dotnet build -t:TestSqlClientManual -p:TestFilters=category=flaky +``` + +Disable the default filter: + +```bash +dotnet build -t:TestSqlClientFunctional -p:TestFilters=none +``` + +When passing filter expressions that contain shell-sensitive characters such as `&`, quote or escape the value as +required by your shell. + +## Running Test Projects Directly + +`build.proj` is the recommended entry point because it keeps logging, code coverage, package-reference mode, and +common parameters consistent. For quick local investigation, you can run a test project directly: + +```bash +dotnet test src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj \ + -p:Configuration=Debug \ + --filter "category!=failing&category!=flaky&category!=interactive" +``` + +For manual tests, pass `TestSet` to the test project when needed: + +```bash +dotnet test src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj \ + -p:Configuration=Debug \ + -p:TestSet=2 \ + --filter "category!=failing&category!=flaky&category!=interactive" +``` + +## Manual Test Prerequisites + +Manual tests require SQL Server or Azure SQL resources and a local test configuration file. + +For a basic local SQL Server run, prepare: + +- A SQL Server instance that the test machine can reach. +- Shared Memory, TCP, and Named Pipes protocols enabled when testing local Windows SQL Server scenarios. +- The `NORTHWIND` database created from [tools/testsql/createNorthwindDb.sql](tools/testsql/createNorthwindDb.sql). For + Azure SQL, use [tools/testsql/createNorthwindAzureDb.sql](tools/testsql/createNorthwindAzureDb.sql). Both scripts turn + `READ_COMMITTED_SNAPSHOT` and `ALLOW_SNAPSHOT_ISOLATION` off, which some transaction tests rely on. Azure SQL Database + enables `READ_COMMITTED_SNAPSHOT` by default, so run the script against the database rather than setting it up by hand. +- The `UdtTestDb` database created from [tools/testsql/createUdtTestDb.sql](tools/testsql/createUdtTestDb.sql) if you + want UDT tests to run. +- A login or integrated-security principal with permissions to create and drop the temporary objects used by the tests. + +Feature-specific tests require additional resources. If those resources are not configured, the corresponding +conditional tests are skipped. + +## Manual Test Configuration + +Edit the source configuration file at `src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/ +config.jsonc`. The test utilities project copies that file to the test output directory, where the manual tests load it +by default. + +The template file is: + +[src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc](src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc) + +`config.jsonc` is git-ignored. If it does not exist, the test utilities project copies `config.default.jsonc` to +`config.jsonc` before compile. You can also create it manually: + +```bash +cp src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc \ + src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.jsonc +``` + +Update `config.jsonc` for your environment before running manual tests. The most important values for a basic run are `TCPConnectionString` and `NPConnectionString`. + +```jsonc +{ + "TCPConnectionString": "Data Source=tcp:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;", + "NPConnectionString": "Data Source=np:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;", + "EnclaveEnabled": false, + "TracingEnabled": false, + "SupportsEntraIntegrated": false, + "SupportsIntegratedSecurity": true +} +``` + +For SQL Server in a Linux container, WSL, or another host where SQL authentication is easier than integrated security, use a TCP connection string like: + +```jsonc +{ + "TCPConnectionString": "Data Source=tcp:127.0.0.1;User Id=sa;Password=;Database=Northwind;Encrypt=false;TrustServerCertificate=true" +} +``` + +You can override the config file path with the `TEST_MDS_CONFIG` environment variable: + +```bash +TEST_MDS_CONFIG=/path/to/config.jsonc dotnet build -t:TestSqlClientManual -p:TestSet=2 +``` + +On PowerShell: + +```powershell +$env:TEST_MDS_CONFIG = "C:\path\to\config.jsonc" +dotnet build -t:TestSqlClientManual -p:TestSet=2 +``` + +## Configuration Properties + +`SupportsEntraIntegrated` applies to Azure SQL Database, Azure SQL Managed Instance, and SQL Server +2022 or later configured for Microsoft Entra authentication through Azure Arc. + +| Property | Description | Example or notes | +|----------------------------------|---------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| +| `TCPConnectionString` | Connection string for a TCP-enabled SQL Server or Azure SQL database. | `Data Source=tcp:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;` | +| `NPConnectionString` | Connection string for a Named Pipes-enabled SQL Server instance. | `Data Source=np:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;` | +| `TCPConnectionStringHGSVBS` | Optional connection string for SQL Server with VBS enclave and HGS attestation. | Include `Attestation Protocol=HGS` and `Enclave Attestation Url`. | +| `TCPConnectionStringNoneVBS` | Optional connection string for SQL Server with VBS enclave and no attestation. | Include `Attestation Protocol=None`. | +| `TCPConnectionStringAASSGX` | Optional connection string for SQL Server with SGX enclave and Microsoft Azure Attestation. | Include `Attestation Protocol=AAS` and `Enclave Attestation Url`. | +| `EnclaveEnabled` | Enables tests that require an enclave-configured server. | `true` or `false`. | +| `TracingEnabled` | Enables tracing-related tests. | `true` or `false`. | +| `AADAuthorityURL` | Optional OAuth authority for `AADPasswordConnectionString`. | `https://login.windows.net/` | +| `AADPasswordConnectionString` | Optional connection string for Microsoft Entra ID password authentication tests. | Uses `Authentication=Active Directory Password`. | +| `AADServicePrincipalId` | Optional application ID for service-principal authentication tests. | Former docs may refer to this as a secure principal ID. | +| `AADServicePrincipalSecret` | Optional application secret for service-principal authentication tests. | Keep this only in local, ignored config files or secure pipeline variables. | +| `AzureKeyVaultURL` | Optional Azure Key Vault URL for Always Encrypted tests. | `https://.vault.azure.net/` | +| `AzureKeyVaultTenantId` | Optional Entra ID tenant ID for Azure Key Vault tests. | Tenant ID GUID. | +| `SupportsEntraIntegrated` | Whether the target supports Entra Integrated authentication for the Windows identity. | `true` or `false`; defaults to `false`. See supported targets above. | +| `SupportsIntegratedSecurity` | Whether the user running tests has integrated-security access to the target SQL Server. | `true` or `false`. | +| `LocalDbAppName` | Optional LocalDB instance name. Empty disables LocalDB testing. | `MSSQLLocalDB` or another local instance. | +| `LocalDbSharedInstanceName` | Optional shared LocalDB instance name. | Used only when testing shared LocalDB. | +| `FileStreamDirectory` | Directory used for FileStream database setup. | Use an escaped absolute path in JSON. | +| `UseManagedSNIOnWindows` | Enables Managed SNI on Windows test coverage. | `true` or `false`. | +| `DNSCachingConnString` | Optional connection string for DNS caching tests. | Used with DNS caching server settings. | +| `DNSCachingServerCR` | Optional DNS caching control-ring server. | Feature-specific tests only. | +| `DNSCachingServerTR` | Optional DNS caching tenant-ring server. | Feature-specific tests only. | +| `IsDNSCachingSupportedCR` | Enables DNS caching control-ring tests. | `true` or `false`. | +| `IsDNSCachingSupportedTR` | Enables DNS caching tenant-ring tests. | `true` or `false`. | +| `EnclaveAzureDatabaseConnString` | Optional Azure SQL database connection string for enclave tests. | Feature-specific tests only. | +| `ManagedIdentitySupported` | Whether managed identity tests should run. | Defaults to `true`. Set `false` if unavailable. | +| `UserManagedIdentityClientId` | Optional client ID for user-assigned managed identity tests. | Feature-specific tests only. | +| `KerberosDomainUser` | Optional Kerberos test domain user. | Feature-specific tests only. | +| `KerberosDomainPassword` | Optional Kerberos test domain password. | Keep only in local, ignored config files or secure pipeline variables. | +| `IsManagedInstance` | Marks the target as Azure SQL Managed Instance. | Set `true` for Managed Instance to use non-Azure TVP baseline files in test set 3. | +| `PowerShellPath` | Full path to PowerShell if it is not on `PATH`. | `C:\\escaped\\path\\to\\powershell.exe` | +| `AliasName` | Optional SQL Server alias used by alias-related tests. | Feature-specific tests only. | + +## Manual Test Sets + +The manual test project is split into compile-time sets so large runs can be parallelized. + +| TestSet | Coverage | +|---------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| `1` | Smaller SQL connectivity and command scenarios. | +| `2` | Broad data access coverage, including adapters, bulk copy, retry logic, data reader, schema, DNS caching, and related scenarios. | +| `3` | Additional integration coverage, including LocalDB, pooling, parameters, transactions, JSON, Kerberos, UDT, vector, and other SQL feature tests. | +| `AE` | Always Encrypted tests. | + +If `TestSet` is omitted, all sets are compiled and run. You can combine sets by concatenating values, for example +`-p:TestSet=23` or `-p:TestSet=12AE`. + +## Results and Diagnostics + +Test results are written to the `test_results` directory by default. Override the location with +`TestResultsFolderPath`: + +```bash +dotnet build -t:TestSqlClientUnit -p:TestResultsFolderPath=/tmp/sqlclient-test-results +``` + +Hang blame collection is enabled by default with a `10m` timeout. To increase the timeout: + +```bash +dotnet build -t:TestSqlClientManual -p:TestBlameTimeout=30m +``` + +To disable hang blame collection: + +```bash +dotnet build -t:TestSqlClientManual -p:TestBlameTimeout=0 +``` + +Code coverage is enabled by default. To disable it for a faster local run: + +```bash +dotnet build -t:TestSqlClientUnit -p:TestCodeCoverage=false +``` diff --git a/build.proj b/build.proj index 33ff1aff31..4dfef2b77b 100644 --- a/build.proj +++ b/build.proj @@ -1,544 +1,1659 @@ - - + + - - + + - - false + + + + + -p:BuildNumber=$(BuildNumber) + + + + + + -p:BuildSuffix=$(BuildSuffix) + + Debug - AnyCPU - - true - - true - false - Windows - Unix - net9.0 - netfx - netcore - netfx - netcoreapp - $(TF) - $(TF) - true - - Configuration=$(Configuration);ReferenceType=$(ReferenceType); - $(CommonProperties);AssemblyFileVersion=$(AssemblyFileVersion);TargetsWindows=$(TargetsWindows);TargetsUnix=$(TargetsUnix); - $(ProjectProperties);BuildForRelease=false;TargetNetCoreVersion=$(TargetNetCoreVersion);TargetNetFxVersion=$(TargetNetFxVersion) - TestResults + - - - DebugType=portable;DebugSymbols=true;IncludeSymbols=true;SymbolPackageFormat=snupkg;PublishRepositoryUrl=true;RepositoryUrl=https://github.com/dotnet/sqlclient;RepositoryType=git;EmbedUnTrackedSources=true;Deterministic=true; - - $(NugetPackProperties);ContinuousIntegrationBuild=true; - + - - - - category!=failing&category!=flaky&category!=interactive - --filter "$(Filter)" - - - true - $(TestsPath)/tools/Microsoft.Data.SqlClient.TestUtilities/CodeCoverage.runsettings - --collect "Code coverage" --settings "$(CodeCoverageRunSettings)" - - - $(Blame) - - --blame-hang - --blame-hang-dump-type full - --blame-hang-timeout 10m - - + + true + + --no-build + - - - true - ContinuousIntegrationBuild=$(BuildForRelease);EmbedUntrackedSources=$(BuildForRelease) - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(CommonProperties) + + + + -p:SqlClientNextVersion=$(SqlClientNextVersion) + - - $(SqlServerProperties);SqlServerPackageVersion=$(SqlServerPackageVersion) - + + + + -p:SqlClientPackageVersion=$(PackageVersionSqlClient) + - - $(SqlServerProperties);SqlServerAssemblyFileVersion=$(SqlServerAssemblyFileVersion) - - + + + + -p:SqlClientFileVersion=$(FileVersionSqlClient) + - - - + + + + -p:SqlServerNextVersion=$(SqlServerNextVersion) + - - - + + + + -p:SqlServerPackageVersion=$(PackageVersionSqlServer) + - - - + + + + -p:SqlServerFileVersion=$(FileVersionSqlServer) + - - - $(CommonProperties) + + Project + + -p:ReferenceType=Package + + false - dotnet build -p:AbstractionsPackageVersion= + + + + --no-incremental + -p:ArtifactPath="$(IsolatedBuildPath)/bin/" + - That results in $(AbstractionsPackageVersion) being defined as empty, - and cannot be overridden by the project. + - - $(AbstractionsProperties);AbstractionsPackageVersion=$(AbstractionsPackageVersion) - - - - - $(AbstractionsProperties);AbstractionsAssemblyFileVersion=$(AbstractionsAssemblyFileVersion) - - + false + + -p:EnableAnalyzers=true + - - - + + false + $(EnableAnalyzersArgument) + -p:InternalAnalyzers=true + - - - + + $(EnableAnalyzersArgument) + -p:RestoreConfigFile="$(InternalAnalyzersNugetConfig)" + - - - + + $(EnableAnalyzersArgument) + -p:InternalAnalyzersVersion=$(InternalAnalyzersVersion) + - - - $(CommonProperties) + + + + -p:SigningKeyPath="$(SigningKeyPath)" + + + + + + -p:TestSigningKeyPath="$(TestSigningKeyPath)" + - - $(LoggingProperties);LoggingPackageVersion=$(LoggingPackageVersion) - - - - - $(LoggingProperties);LoggingAssemblyFileVersion=$(LoggingAssemblyFileVersion) - + 10m + + --blame-hang + --blame-hang-dump-type full + --blame-hang-timeout $(TestBlameTimeout) + + + + true + + --collect "Code coverage" + --settings "$(RepoRoot)src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/CodeCoverage.runsettings" + + + + category!=failing&category!=flaky&category!=interactive + + + $(TestFilters)&category!=signed + + --filter "$(TestFilters)" + + + + + -f $(TestFramework) + + + $(RepoRoot)test_results + + + - - + + + + + + + + + + + + + + + + + + - - + + + + + + + + + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlClientNextVersionArgument) + <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) + + + + + + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlClientNextVersionArgument) + <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) + + + + + + - - + + + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlServerNextVersionArgument) + <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) + + + + + + + + + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlServerNextVersionArgument) + <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) + + + + + + + - - - $(CommonProperties) + + + + + + + + $(RepoRoot)src/Microsoft.Data.SqlClient/ + $(RepoRoot)artifacts/Microsoft.Data.SqlClient/ + + + $(SqlClientSrcRoot)src/Microsoft.Data.SqlClient.csproj + $(SqlClientSrcRoot)ref/Microsoft.Data.SqlClient.csproj + $(SqlClientSrcRoot)notsupported/Microsoft.Data.SqlClient.csproj + $(SqlClientArtifactRoot)$(ReferenceType)-$(Configuration)/ + + + $(SqlClientSrcRoot)tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj + $(SqlClientSrcRoot)tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj + $(SqlClientSrcRoot)tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj + + + $(RepoRoot)tools/GenAPI/Microsoft.DotNet.GenAPI/ + $(GenApiPath)Microsoft.DotNet.GenAPI.csproj + - dotnet build -p:AzurePackageVersion= + + + - That results in $(AzurePackageVersion) being defined as empty, and cannot - be overridden by the project. - --> - - $(AzureProperties);AzurePackageVersion=$(AzurePackageVersion) - - - - - $(AzureProperties);AzureAssemblyFileVersion=$(AzureAssemblyFileVersion) - + + + PackLogging;PackAbstractions;PackSqlServer + + + + "$(DotnetPath)dotnet" build "$(GenApiProjectPath)" + -p:Configuration=$(Configuration) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(SqlClientNotSupportedProjectPath)" + -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + -p:GenApiPath="@(GenApiArtifactPath->'%(FullPath)')" + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + - - + + + - - + + + PackLogging;PackAbstractions;PackSqlServer + + + + + "$(DotnetPath)dotnet" build $(SqlClientRefProjectPath) + -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + - - + + + PackLogging;PackAbstractions;PackSqlServer + + + + + "$(DotnetPath)dotnet" build $(SqlClientProjectPath) + -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + - - - + + + + BuildSqlClient + + + + + + + "$(DotnetPath)dotnet" pack "$(SqlClientProjectPath)" + -p:Configuration=$(Configuration) + $(PackBuildArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlServerArgument) + + + -p:PackageOutputPath="$(SqlClientPackageArtifactRoot)" + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + - - + + + + + + + + + SqlClientFunctional-$(OS) + $(LogFilePrefix)-$(TestFramework) + + + "$(DotnetPath)dotnet" test "$(SqlClientFunctionalTestProjectPath)" + -p:Configuration=$(Configuration) + $(TestBlameArgument) + $(TestCodeCoverageArgument) + $(TestFiltersArgument) + $(TestFrameworkArgument) + --results-directory "$(TestResultsFolderPath)" + --logger:"trx;LogFilePrefix=$(LogFilePrefix)" + + + $(ReferenceTypeArgument) + $(PackageVersionSqlClientArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + - - + + + + SqlClientManual-$(OS) + $(LogFilePrefix)-$(TestFramework) + $(LogFilePrefix)-$(TestSet) + + + $(TestSet.ToUpper()) + Set=1 + $(TestSetFilter)|Set=2 + $(TestSetFilter)|Set=3 + $(TestSetFilter)|Set=AE + $(TestSetFilter.Trim('|')) + + + $(TestFilters)&($(TestSetFilter)) + $(TestFilters) + $(TestSetFilter) + --filter "$(ManualTestFilters)" + + + "$(DotnetPath)dotnet" test "$(SqlClientManualTestProjectPath)" + -p:Configuration=$(Configuration) + $(TestBlameArgument) + $(TestCodeCoverageArgument) + $(ManualTestFiltersArgument) + $(TestFrameworkArgument) + --results-directory "$(TestResultsFolderPath)" + --logger:"trx;LogFilePrefix=$(LogFilePrefix)" + + + $(ReferenceTypeArgument) + $(PackageVersionSqlClientArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + - - + + + + SqlClientUnit-$(OS) + $(LogFilePrefix)-$(TestFramework) + + + "$(DotnetPath)dotnet" test "$(SqlClientUnitTestProjectPath)" + -p:Configuration=$(Configuration) + $(TestBlameArgument) + $(TestCodeCoverageArgument) + $(TestFiltersArgument) + $(TestFrameworkArgument) + $(ReferenceTypeArgument) + $(TestSigningKeyPathArgument) + $(PackageVersionSqlClientArgument) + $(PackageVersionSqlServerArgument) + --results-directory "$(TestResultsFolderPath)" + --logger:"trx;LogFilePrefix=$(LogFilePrefix)" + + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + - - + + + + $(RepoRoot)src/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/ + $(AkvProviderSrcRoot)src/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.csproj + $(AkvProviderSrcRoot)test/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.Test.csproj + $(RepoRoot)artifacts/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/$(Configuration)/ + + + + + PackSqlClient;PackLogging + + + + + "$(DotnetPath)dotnet" build "$(AkvProviderProjectPath)" + -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + - - + + + + + "$(DotnetPath)dotnet" pack "$(AkvProviderProjectPath)" + -p:Configuration=$(Configuration) + $(PackBuildArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + + - - - - + + + + + AkvProviderTests-$(OS) + $(LogFilePrefix)-$(TestFramework) + + + "$(DotnetPath)dotnet" test "$(AkvProviderTestProjectPath)" + -p:Configuration=$(Configuration) + $(TestBlameArgument) + $(TestCodeCoverageArgument) + $(TestFiltersArgument) + $(TestFrameworkArgument) + --results-directory "$(TestResultsFolderPath)" + --logger:"trx;LogFilePrefix=$(LogFilePrefix)" + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + - - - + + + + $(RepoRoot)src/Microsoft.Data.SqlClient.Extensions/Abstractions/ + $(AbstractionsSrcRoot)src/Abstractions.csproj + $(AbstractionsSrcRoot)test/Abstractions.Test.csproj + $(RepoRoot)artifacts/Microsoft.Data.SqlClient.Extensions.Abstractions/$(Configuration)/ + - - - + + + PackLogging + + + + + "$(DotnetPath)dotnet" build "$(AbstractionsProjectPath)" + -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + - + + + + + "$(DotnetPath)dotnet" pack "$(AbstractionsProjectPath)" + -p:Configuration=$(Configuration) + $(PackBuildArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + - - + + + - - + + + + + + + - - + + - - $(DotnetPath)dotnet test "@(UnitTestsProj)" - -f $(TF) - -p:Configuration=$(Configuration) - $(FilterArgument) - $(BlameArgument) - $(CollectArgument) - --results-directory $(ResultsDirectory) - --logger:"trx;LogFilePrefix=Unit-Windows$(TargetGroup)-$(TestSet)" - + AbstractionsTests-$(OS) + $(LogFilePrefix)-$(TestFramework) + + + "$(DotnetPath)dotnet" test "$(AbstractionsTestProjectPath)" + -p:Configuration=$(Configuration) + $(TestBlameArgument) + $(TestCodeCoverageArgument) + $(TestFiltersArgument) + $(TestFrameworkArgument) + --results-directory "$(TestResultsFolderPath)" + --logger:"trx;LogFilePrefix=$(LogFilePrefix)" + - $([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " ")) + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) - - + + + + - - + + + + $(RepoRoot)src/Microsoft.Data.SqlClient.Extensions/Azure/ + $(AzureSrcRoot)src/Azure.csproj + $(AzureSrcRoot)test/Azure.Test.csproj + $(RepoRoot)artifacts/Microsoft.Data.SqlClient.Extensions.Azure/$(Configuration)/ + + + + + PackLogging;PackAbstractions + + - - $(DotnetPath)dotnet test "@(UnitTestsProj)" - -f $(TF) + + "$(DotnetPath)dotnet" build "$(AzureProjectPath)" -p:Configuration=$(Configuration) - $(FilterArgument) - $(BlameArgument) - $(CollectArgument) - --results-directory $(ResultsDirectory) - --logger:"trx;LogFilePrefix=Unit-Unixnetcoreapp-$(TestSet)" - + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + - $([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " ")) + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) - - - - - + + + + - - + + - - $(DotnetPath)dotnet test "@(FunctionalTestsProj)" - -f $(TF) + + "$(DotnetPath)dotnet" pack "$(AzureProjectPath)" -p:Configuration=$(Configuration) - -p:ReferenceType=$(ReferenceType) - -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion) - -p:MdsPackageVersion=$(MdsPackageVersion) - $(FilterArgument) - $(BlameArgument) - $(CollectArgument) - --results-directory $(ResultsDirectory) - --logger:"trx;LogFilePrefix=Functional-Windows$(TargetGroup)-$(TestSet)" - + $(PackBuildArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $(ReferenceTypeArgument) + - $([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " ")) + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) - - + + + + + + + + + + + - - + + - - $(DotnetPath)dotnet test "@(FunctionalTestsProj)" - -f $(TF) + AzureTests-$(OS) + $(LogFilePrefix)-$(TestFramework) + + + "$(DotnetPath)dotnet" test "$(AzureTestProjectPath)" -p:Configuration=$(Configuration) - -p:ReferenceType=$(ReferenceType) - -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion) - -p:MdsPackageVersion=$(MdsPackageVersion) - $(FilterArgument) - $(BlameArgument) - $(CollectArgument) - --results-directory $(ResultsDirectory) - --logger:"trx;LogFilePrefix=Functional-Unixnetcoreapp-$(TestSet)" - + $(TestBlameArgument) + $(TestCodeCoverageArgument) + $(TestFiltersArgument) + $(TestFrameworkArgument) + --results-directory "$(TestResultsFolderPath)" + --logger:"trx;LogFilePrefix=$(LogFilePrefix)" + + + $(ReferenceTypeArgument) + $(PackageVersionSqlClientArgument) + $(PackageVersionSqlServerArgument) + - $([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " ")) + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) - - + + + + - - + + + + $(RepoRoot)src/Microsoft.Data.SqlClient.Internal/Logging/src/ + $(LoggingSrcRoot)Logging.csproj + $(RepoRoot)artifacts/Microsoft.Data.SqlClient.Internal.Logging/$(Configuration)/ + - - + + - - $(DotnetPath)dotnet test "@(ManualTestsProj)" - -f $(TF) + + "$(DotnetPath)dotnet" build $(LoggingProjectPath) -p:Configuration=$(Configuration) - -p:ReferenceType=$(ReferenceType) - -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion) - -p:MdsPackageVersion=$(MdsPackageVersion) - $(FilterArgument) - $(BlameArgument) - $(CollectArgument) - --results-directory $(ResultsDirectory) - --logger:"trx;LogFilePrefix=Manual-Windows$(TargetGroup)-$(TestSet)" - - - - $(TestCommand) - -p:TestSet=$(TestSet) - - + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + - $([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " ")) + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) - - + + + + - - + + - - $(DotnetPath)dotnet test "@(ManualTestsProj)" - -f $(TF) + + "$(DotnetPath)dotnet" pack $(LoggingProjectPath) -p:Configuration=$(Configuration) - -p:ReferenceType=$(ReferenceType) - -p:AbstractionsPackageVersion=$(AbstractionsPackageVersion) - -p:MdsPackageVersion=$(MdsPackageVersion) - $(FilterArgument) - $(BlameArgument) - $(CollectArgument) - --results-directory $(ResultsDirectory) - --logger:"trx;LogFilePrefix=Manual-Unixnetcoreapp-$(TestSet)" - + $(PackBuildArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + + + - - - $(TestCommand) - -p:TestSet=$(TestSet) - + + + + $(RepoRoot)src/Microsoft.SqlServer.Server/ + $(SqlServerSrcRoot)Microsoft.SqlServer.Server.csproj + $(RepoRoot)artifacts/Microsoft.SqlServer.Server/$(Configuration)/ + + + + + + "$(DotnetPath)dotnet" build $(SqlServerProjectPath) + -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlServerArgument) + $(FileVersionSqlServerArgument) + - $([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " ")) + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) - - + + + + - - - - - - - - - - - - + + + + + "$(DotnetPath)dotnet" pack $(SqlServerProjectPath) + -p:Configuration=$(Configuration) + $(PackBuildArgument) + $(SigningKeyPathArgument) + + + $(BuildNumberArgument) + $(BuildSuffixArgument) + $(PackageVersionSqlServerArgument) + $(FileVersionSqlServerArgument) + + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + - + - + + + - $(CommonProperties) - - - $(AkvProviderProperties);AkvPackageVersion=$(AkvPackageVersion) - - - - $(AkvProviderProperties);AkvAssemblyFileVersion=$(AkvAssemblyFileVersion) - - - - $(AkvProviderProperties);MdsPackageVersion=$(MdsPackageVersion) - - - - $(AkvProviderProperties);LoggingPackageVersion=$(LoggingPackageVersion) - - - - $(AkvProviderProperties);AbstractionsPackageVersion=$(AbstractionsPackageVersion) - + $(SqlClientSrcRoot)tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj + $(SqlClientSrcRoot)tests/StressTests/SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj + $(RepoRoot)doc/samples/Microsoft.Data.SqlClient.Samples.csproj - - + + + + + + + + "$(DotnetPath)dotnet" build "$(AkvProviderTestProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + - - + + + + + "$(DotnetPath)dotnet" build "$(AbstractionsTestProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + - - + + + + + "$(DotnetPath)dotnet" build "$(SqlClientUnitTestProjectPath)" + -p:Configuration=$(Configuration) + + + $(ReferenceTypeArgument) + $(TestSigningKeyPathArgument) + $(PackageVersionSqlClientArgument) + $(PackageVersionSqlServerArgument) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(SqlClientFunctionalTestProjectPath)" + -p:Configuration=$(Configuration) + + + $(ReferenceTypeArgument) + $(PackageVersionAbstractionsArgument) + $(PackageVersionLoggingArgument) + $(PackageVersionSqlClientArgument) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(SqlClientManualTestProjectPath)" + -p:Configuration=$(Configuration) + + + $(ReferenceTypeArgument) + $(PackageVersionAbstractionsArgument) + $(PackageVersionLoggingArgument) + $(PackageVersionSqlClientArgument) + $(PackageVersionSqlServerArgument) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(SqlClientPerformanceTestProjectPath)" + -p:Configuration=$(Configuration) + + + $(ReferenceTypeArgument) + $(PackageVersionAbstractionsArgument) + $(PackageVersionLoggingArgument) + $(PackageVersionSqlClientArgument) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(SqlClientStressTestProjectPath)" + -p:Configuration=$(Configuration) + + + $(ReferenceTypeArgument) + $(PackageVersionAbstractionsArgument) + $(PackageVersionLoggingArgument) + $(PackageVersionSqlClientArgument) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(AzureTestProjectPath)" + -p:Configuration=$(Configuration) + + + $(ReferenceTypeArgument) + $(PackageVersionSqlClientArgument) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(SamplesProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + $(RepoRoot)doc/apps/AzureSqlConnector/AzureSqlConnector.csproj + $(RepoRoot)tools/PackageCompatibility/src/PackageCompatibility.csproj + $(RepoRoot)tools/PackageCompatibility/test/PackageCompatibility.Test.csproj + $(RepoRoot)tools/PackageCompatibility + $(RepoRoot)tools/PackageValidator/src/PackageValidator.csproj + $(RepoRoot)tools/PackageValidator/test/PackageValidator.Test.csproj + $(RepoRoot)tools/PackageValidator + + + + + + + + + + "$(DotnetPath)dotnet" build "$(AzureSqlConnectorProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(PackageCompatibilityProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(PackageValidatorProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(PackageCompatibilityTestProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" build "$(PackageValidatorTestProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" test "$(PackageCompatibilityTestProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + + + + + + + + "$(DotnetPath)dotnet" test "$(PackageValidatorTestProjectPath)" + -p:Configuration=$(Configuration) + + $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) + + + + diff --git a/build2.proj b/build2.proj deleted file mode 100644 index a5fb6b89a2..0000000000 --- a/build2.proj +++ /dev/null @@ -1,400 +0,0 @@ - - - - - - - - - - Debug - - - - - - - - -p:AbstractionsPackageVersion=$(PackageVersionAbstractions) - - - - - - -p:LoggingPackageVersion=$(PackageVersionLogging) - - - - - - -p:MdsPackageVersion=$(PackageVersionMds) - - - - Project - - -p:ReferenceType=Package - - - - 10m - - --blame-hang - --blame-hang-dump-type full - --blame-hang-timeout $(TestBlameTimeout) - - - - true - - --collect "Code coverage" - --settings "$(RepoRoot)src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/CodeCoverage.runsettings" - - - - category!=failing&category!=flaky - - - --filter "$(TestFilters)" - - - - - -f $(TestFramework) - - - $(RepoRoot)test_results - - - - -p:TestSet="$(TestSet)" - - - - - - - $(RepoRoot)src/Microsoft.Data.SqlClient/ - - - $(RepoRoot)tools/specs/Microsoft.Data.SqlClient.nuspec - $(RepoRoot)tools/GenAPI/Microsoft.DotNet.GenAPI/ - $(GenApiPath)Microsoft.DotNet.GenAPI.csproj - - - $(MdsRoot)tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj - $(MdsRoot)tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj - $(MdsRoot)notsupported/Microsoft.Data.SqlClient.csproj - $(MdsRoot)src/Microsoft.Data.SqlClient.csproj - $(MdsRoot)ref/Microsoft.Data.SqlClient.csproj - $(MdsRoot)tests/UnitTests/Microsoft.Data.SqlClient.UnitTests.csproj - - - - - - - - - - - "$(DotNetPath)dotnet" build "$(GenApiProjectPath)" - -p:Configuration=$(Configuration) - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - - - - - "$(DotNetPath)dotnet" build "$(MdsNotSupportedProjectPath)" - -p:Configuration=$(Configuration) - -p:GenApiPath="@(GenApiArtifactPath->'%(FullPath)')" - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - - - $(DotNetPath)dotnet build $(MdsRefProjectPath) - -p:Configuration=$(Configuration) - $(ReferenceTypeArgument) - $(PackageVersionAbstractionsArgument) - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - - - $(DotNetPath)dotnet build $(MdsProjectPath) - -p:Configuration=$(Configuration) - -p:TargetOs=Unix - - - $(ReferenceTypeArgument) - $(PackageVersionAbstractionsArgument) - $(PackageVersionLoggingArgument) - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - - - $(DotNetPath)dotnet build $(MdsProjectPath) - -p:Configuration=$(Configuration) - -p:TargetOs=Windows_NT - - - $(ReferenceTypeArgument) - $(PackageVersionAbstractionsArgument) - $(PackageVersionLoggingArgument) - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - - - - - - - - - - - - MdsFunctional-$(OS) - $(LogFilePrefix)-$(TestFramework) - - - $(DotNetPath)dotnet test "$(MdsFunctionalTestProjectPath)" - -p:Configuration=$(Configuration) - $(TestBlameArgument) - $(TestCollectArgument) - $(TestFiltersArgument) - $(TestFrameworkArgument) - --results-directory "$(TestResultsFolderPath)" - --logger:"trx;LogFilePrefix=$(LogFilePrefix)" - - - $(ReferenceTypeArgument) - $(PackageVersionAbstractionsArgument) - $(PackageVersionLoggingArgument) - $(PackageVersionMdsArgument) - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - MdsManual-$(OS) - $(LogFilePrefix)-$(TestFramework) - $(LogFilePrefix)-$(TestSet) - - - $(DotNetPath)dotnet test "$(MdsManualTestProjectPath)" - -p:Configuration=$(Configuration) - $(TestBlameArgument) - $(TestCollectArgument) - $(TestFiltersArgument) - $(TestFrameworkArgument) - $(TestSetArgument) - --results-directory "$(TestResultsFolderPath)" - --logger:"trx;LogFilePrefix=$(LogFilePrefix)" - - - $(ReferenceTypeArgument) - $(PackageVersionAbstractionsArgument) - $(PackageVersionLoggingArgument) - $(PackageVersionMdsArgument) - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - - - - - - MdsUnit-$(OS) - $(LogFilePrefix)-$(TestFramework) - - - $(DotNetPath)dotnet test "$(MdsUnitTestProjectPath)" - -p:Configuration=$(Configuration) - $(TestBlameArgument) - $(TestCollectArgument) - $(TestFiltersArgument) - $(TestFrameworkArgument) - --results-directory "$(TestResultsFolderPath)" - --logger:"trx;LogFilePrefix=$(LogFilePrefix)" - - - - $([System.Text.RegularExpressions.Regex]::Replace($(DotNetCommand), "\s+", " ")) - - - - - diff --git a/contributing-workflow.md b/contributing-workflow.md index 266c39c1a7..be7fffdbf2 100644 --- a/contributing-workflow.md +++ b/contributing-workflow.md @@ -7,7 +7,7 @@ You can contribute to Microsoft.Data.SqlClient with issues and PRs. Simply filin We use and recommend the following workflow: 1. **Create an issue for your work.** - - You can skip this step for trivial changes. + - **For all significant changes you intend to make in this repository, an issue is required.** This gives maintainers a chance to review the proposal before implementation begins and prevents wasted effort. Bug fixes and small improvements are generally welcome without a prior issue, but a linked issue helps us triage and prioritize your contribution. - Reuse an existing issue on the topic, if there is one. - Get agreement from the team and the community that your proposed change is a good one. - Suggest [Labels](CONTRIBUTING.md#using-labels) to add for your issue. @@ -39,11 +39,12 @@ We use and recommend the following workflow: - It is OK to create your PR as Draft on the upstream repo before the implementation is done. - This can be useful if you'd like to start the feedback process concurrent with your implementation. - State that this is the case in the initial PR comment. +- **Keep your PR in Draft until CI validation passes.** The PR pipeline runs without maintainer intervention, so contributors should ensure all checks are green before marking the PR as ready for review. PRs appearing ready for review with failing checks will not be picked up by reviewers. ### Commit Guidelines - It is OK for your PR to include a large number of commits. -- Once your change is accepted, you will be asked to squash your commits into one or some appropriately small number of commits before your PR is merged. +- PRs are merged via the default "Squash and Merge" strategy (see [Merging Pull Requests](#merging-pull-requests)), so your commit history is automatically collapsed into a single commit at merge time. ## PR - CI Process @@ -63,13 +64,98 @@ If the CI build fails for any reason, the PR issue will be updated with a link t - **Build failures**: Check the CI logs for specific error messages. - **Test failures**: Ensure all tests pass locally before pushing. -- **Merge conflicts**: Rebase your branch against the latest main branch. +- **Merge conflicts**: + - *Before review has started*: Rebase your branch against the latest `main` branch to keep history clean. + - *After a reviewer has started reviewing*: Do **not** rebase. Rebasing rewrites commit history and makes it impossible for reviewers to tell which commits they have already reviewed, forcing them to start over. Instead, resolve conflicts by merging the latest `main` into your branch (`git merge main`) so that previously reviewed commits remain intact and reviewers can pick up where they left off. ### Getting Help - Comment on your PR if you're stuck. - Reach out in [GitHub Discussions](https://github.com/dotnet/SqlClient/discussions) for broader questions. +## Tracking PRs in the GitHub Project + +The [SqlClient Board](https://github.com/orgs/dotnet/projects/588) (dotnet org project #588) is the team's triage board for tracking issues and PRs. When a PR is linked to an issue, its progress is tracked through the project's fields described below. + +### Project Fields + +| Field | Type | Values | Purpose | +|-------|------|--------|---------| +| **Status** | Single select | `To triage`, `Waiting for customer`, `In progress`, `In review`, `In validation`, `Done` | Tracks the current workflow stage of the item | +| **Priority** | Single select | `P0`, `P1`, `P2`, `P3` | Indicates urgency and scheduling priority | +| **Size** | Single select | `XS`, `S`, `M`, `L`, `XL` | Estimates effort/complexity of the work | +| **API Impact** | Single select | `Breaking Change`, `New API`, `None` | Flags whether the change affects public API surface | +| **PM Approved** | Single select | `N/A`, `Pending`, `Approved` | Tracks product manager sign-off for API or behavioral changes | +| **Assignees** | Field | *(GitHub users)* | Who is responsible for the work | +| **Labels** | Field | *(GitHub labels)* | Categorization and area tags | +| **Reviewers** | Field | *(GitHub users)* | Assigned code reviewers | +| **Milestone** | Field | *(GitHub milestones)* | Target release version | +| **Linked pull requests** | Field | *(PR references)* | PRs associated with the issue | +| **Parent issue** | Field | *(issue reference)* | Epic or parent tracking issue | +| **Sub-issues progress** | Field | *(auto-calculated)* | Completion progress of child issues | +| **Comment** | Text | *(free text)* | Additional context or notes | +| **AB-Link** | Text | *(URL)* | Link to Azure Boards work item (if applicable) | + +### Status Stages for PRs + +PRs (and their linked issues) move through the **Status** field to communicate reviewability and progress: + +| Status | What it means for a PR | +|--------|------------------------| +| **To triage** | PR just opened or linked issue is awaiting initial assessment. Maintainers will review scope, assign reviewers, and set priority. | +| **In progress** | Author is actively developing the change. PR may be in Draft state. | +| **In review** | PR is ready for code review. Reviewers are assigned and the author considers the implementation complete. | +| **Waiting for customer** | Review feedback has been given; the PR author needs to respond or push changes. When a reviewer requests changes, they will also apply the **`Author attention needed`** label so the PR is easy to surface in queries and dashboards. Once the author addresses the feedback, they should remove the `Author attention needed` label if permissions allow or comment `/ready` on the PR to remove the label `Author attention needed` and re-engage the reviewers to move the PR back to `In review`. | +| **In validation** | PR is approved and being validated (CI, manual testing, integration checks) before merge. | +| **Done** | PR has been merged and the associated work is complete. | + +### Status Transitions + +When a reviewer leaves feedback that needs author action, they move the PR to `Waiting for customer` **and** apply the `Author attention needed` label. The author removes the label (and updates Status back to `In review`/`In progress`) once the feedback is addressed. + +``` ++--------------+ +---------------+ +-------------------------+ +-----------------+ +--------+ +| To triage |------>| In progress |------>| In review |------>| In validation |------>| Done | ++--------------+ +---------------+ +------------+------------+ +-----------------+ +--------+ + ^ | ^ + | | changes | author addresses + | | requested | feedback (minor) + + | | + label | removes label + | v | + | +------------------------+ + | | Waiting for customer | + | | + Author attention | + | | needed (label) | + | +------------------------+ + | | + | | major rework needed + +----------------------+ +``` + +### Additional Field Usage + +- **Priority**: `P0` items are critical fixes that should be reviewed and merged urgently. `P1` items are high priority for the current milestone. `P2`/`P3` items are scheduled as capacity allows. +- **Size**: Helps reviewers estimate review effort. `XS`/`S` PRs should get faster turnaround. `L`/`XL` PRs may need multiple reviewers or phased review. +- **API Impact**: PRs marked `Breaking Change` or `New API` require **PM Approved = Approved** before merge. `ref/` project updates must accompany these PRs. +- **PM Approved**: Set to `Pending` when a PR introduces API changes. Must reach `Approved` before the PR can be merged. `N/A` for internal or non-API changes. + +### Guidelines for Contributors + +> **Note:** Setting labels and project board fields (Status, Priority, etc.) requires maintainer or triage access. External contributors cannot modify these directly. Instead, suggest labels in your PR description and the team will apply them. If review feedback asks you to "remove the label" or "update Status", a maintainer will handle it if you don't have access—just let the team know in a comment. + +1. **Link your PR to an issue** — This ensures the PR appears on the project board and is tracked through the workflow. +2. **Keep Status current** — If you're the author, move your linked issue to `In review` when your PR is ready for feedback. +3. **Respond promptly** — When status is `Waiting for customer` and the **`Author attention needed`** label is applied, the team is blocked on your response. Address the feedback, push the updates, **remove the `Author attention needed` label**, and move Status back to `In review` (or `In progress` for major rework) so reviewers know the PR is ready for another pass. If you are a community contributor, you may post a comment `/ready` on the PR to automate removing the label and re-engaging reviewers on the PR. +4. **Flag API changes early** — Set **API Impact** appropriately so PM review can happen in parallel with code review. +5. **Don't skip validation** — Even after approval, the PR stays in `In validation` until CI passes and any manual verification is complete. + +### Guidelines for Reviewers + +1. **Signal that author action is required** — When your review requests changes (whether via formal "Request changes" or a comment that requires author follow-up), apply the **`Author attention needed`** label to the PR and move its linked issue to `Waiting for customer`. This pair (label + status) is what the team uses to find PRs that are blocked on the author versus those still awaiting review. +2. **Don't remove the label yourself** — Leave the `Author attention needed` label in place until the author pushes updates addressing the feedback; the author is responsible for removing it when they hand the PR back for review. +3. **Re-review promptly** — Once the author removes the label, the PR status should be updated either by Author (if they have permissions) or the reviewer to `In review`, so it doesn't stall and is picked up for review on time. +4. **Only reviewers resolve their own feedback threads** — A review comment thread should only be resolved by the reviewer who created it, once they are satisfied the feedback has been addressed. Authors should not resolve reviewer threads themselves. + ## Stale PR Management The SqlClient repository uses automated workflows to manage inactive pull requests and maintain repository hygiene. @@ -111,6 +197,7 @@ One or more Microsoft team members will review every PR prior to merge. They wil - Address all review comments before the PR can be merged. - Feel free to ask questions if feedback is unclear. - Push additional commits to address feedback (these will be squashed later). +- When a reviewer leaves feedback that needs your action, they will set Status to `Waiting for customer` and apply the **`Author attention needed`** label. After you push updates, **remove the `Author attention needed` label** or post a comment `/ready` to do so, so the reviewers know the PR is ready for another pass. ## Merging Pull Requests diff --git a/doc/Directory.Packages.props b/doc/Directory.Packages.props index 2ead6c1757..d47f7e01b9 100644 --- a/doc/Directory.Packages.props +++ b/doc/Directory.Packages.props @@ -6,5 +6,6 @@ + diff --git a/doc/apps/AzureAuthentication/Directory.Packages.props b/doc/apps/AzureAuthentication/Directory.Packages.props deleted file mode 100644 index b512683765..0000000000 --- a/doc/apps/AzureAuthentication/Directory.Packages.props +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - 7.0.0-preview4.26064.3 - - - 7.0.0-preview1.26064.3 - - - - - - - - - - - - - - - - - - diff --git a/doc/apps/AzureAuthentication/NuGet.config b/doc/apps/AzureAuthentication/NuGet.config deleted file mode 100644 index 1c814bbc3c..0000000000 --- a/doc/apps/AzureAuthentication/NuGet.config +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - diff --git a/doc/apps/AzureAuthentication/README.md b/doc/apps/AzureAuthentication/README.md deleted file mode 100644 index a0d9d8b7ba..0000000000 --- a/doc/apps/AzureAuthentication/README.md +++ /dev/null @@ -1,165 +0,0 @@ -# AzureAuthentication Sample App - -A minimal console application that verifies **SqlClient** can connect to a SQL Server using Entra ID -authentication (formerly Azure Active Directory authentication) via the **Azure** package. It also -references the **Azure Key Vault Provider** package to confirm there are no transitive dependency -conflicts between the packages. - -The following SqlClient packages are used, either directly or transitively: - -- `Microsoft.Data.SqlClient` -- `Microsoft.SqlServer.Server` -- `Microsoft.Data.SqlClient.Internal.Logging` -- `Microsoft.Data.SqlClient.Extensions.Abstractions` -- `Microsoft.Data.SqlClient.Extensions.Azure` -- `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` - -## Purpose - -This app serves as a smoke test for package compatibility. It: - -1. Instantiates `SqlColumnEncryptionAzureKeyVaultProvider` to ensure the AKV provider assembly loads - without conflicts. -2. Opens a `SqlConnection` using a connection string you provide, validating that authentication and - connectivity work end-to-end. - -The app is designed to run against both **published NuGet packages** and **locally-built packages** -(via the `packages/` directory configured in `NuGet.config`). - -## Build Parameters - -Package versions are controlled through MSBuild properties. Pass them on the command line with `-p:` -(or `/p:`) to override the defaults defined in `Directory.Packages.props`. - -| Property | Default | Description | -| --- | --- | --- | -| `SqlClientVersion` | `7.0.0-preview4.26064.3` | Version of `Microsoft.Data.SqlClient` to reference. | -| `AkvProviderVersion` | `7.0.0-preview1.26064.3` | Version of `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` to reference. | -| `AzureVersion` | None | Version of `Microsoft.Data.SqlClient.Extensions.Azure` to reference. When omitted, the `Azure` package will not be referenced. | - -## Local Package Source - -The `NuGet.config` adds a `packages/` directory as a local package source. To test against packages -that haven't been published to NuGet yet, copy the `.nupkg` files into this folder and specify the -matching version via the build properties above. - -NuGet will cache copies of the packages it finds in `packages/` after a successful restore. If you -update the `.nupkg` files in `packages/` without incrementing their version numbers (and referencing -those new version numbers) you will have to clear the NuGet caches in order for the next restore -operation to pick them up: - -```bash -dotnet nuget locals all --clear -``` - -## Running the App - -The app has built-in help: - -```bash -dotnet run -- --help - -Description: - Azure Authentication Tester - --------------------------- - - Validates SqlClient connectivity using EntraID (formerly Azure Active Directory) authentication. - Connects to SQL Server using the supplied connection string, which must specify the authentication method. - - Supply specific package versions when building to test different versions of the SqlClient suite, for example: - - -p:SqlClientVersion=7.0.0.preview4 - -p:AkvProviderVersion=7.0.1-preview2 - -p:AzureVersion=1.0.0-preview1 - -Usage: - AzureAuthentication [options] - -Options: - -c, --connection-string (REQUIRED) The ADO.NET connection string used to connect to SQL Server. - Supports SQL, Entra ID, and integrated authentication modes. - -l, --log-events Enable SqlClient event emission to the console. - -t, --trace Pauses execution to allow dotnet-trace to be attached. - -v, --verbose Enable verbose output with detailed error information. - -?, -h, --help Show help and usage information - --version Show version information -``` - -The app expects a single argument: a full connection string. - -```bash -dotnet run -- -c "" -``` - -For Entra ID authentication, use an `Authentication` keyword in the connection string. For example: - -```bash -dotnet run -- -c "Server=myserver.database.windows.net;Database=mydb;Authentication=ActiveDirectoryDefault" -``` - -On success the app emits to standard out: - -```bash -Azure Authentication Tester ---------------------------- - -Packages used: - SqlClient: 7.0.0-preview4.26055.1 - AKV Provider: 6.1.2 - Azure: 1.0.0-preview1.26055.1 - -Connection details: - Data Source: adotest.database.windows.net - Initial Catalog: Northwind - Authentication: ActiveDirectoryPassword - -Testing connectivity... -Connected successfully! - Server version: 12.00.1017 -``` - -Errors will be emitted to standard error: - -```bash -Testing connectivity... -Connection failed: - Cannot find an authentication provider for 'ActiveDirectoryPassword'. -``` - -### Examples - -Run with the default (published) package versions, and no `Azure` package: - -```bash -dotnet run -- -c "" -``` - -If the connection string specifies one of the Entra ID authentication methods, -`SqlClient` will fail with an error indicating that no authentication provider has been registered. -This is because the `Azure` package was not referenced, and the app did not provide its own custom -authentication provider. - -Run against locally-built packages (drop `.nupkg` files into the `packages/` folder first): - -```bash -dotnet run -p:SqlClientVersion=7.0.0-preview4 -- -c "" -``` - -Run including the `Azure` extensions package: - -```bash -dotnet run -p:AzureVersion=1.0.0-preview1 -- -c "" -``` - -Override all three versions at once: - -```bash -dotnet run -p:SqlClientVersion=7.0.0-preview1 -p:AkvProviderVersion=7.0.0-preview1 -p:AzureVersion=1.0.0-preview1 -- -c "" -``` - -## Prerequisites - -- [.NET 10.0 SDK](https://dotnet.microsoft.com/download) and .NET Framework 4.8.1 or later. -- A SQL Server or Azure SQL instance accessible with Entra ID credentials. -- Azure credentials available to `DefaultAzureCredential` (e.g. Azure CLI login, environment - variables, or managed identity). diff --git a/doc/apps/AzureSqlConnector/AzureSqlConnector.csproj b/doc/apps/AzureSqlConnector/AzureSqlConnector.csproj new file mode 100644 index 0000000000..0ea12bd7d2 --- /dev/null +++ b/doc/apps/AzureSqlConnector/AzureSqlConnector.csproj @@ -0,0 +1,34 @@ + + + + + net481;net10.0-windows + net10.0-windows + + true + WinExe + Microsoft.Data.SqlClient.Samples.AzureSqlConnector + AzureSqlConnector + true + latest + disable + AnyCPU + true + false + + + + + + + + diff --git a/doc/apps/AzureSqlConnector/IdentityQuery.cs b/doc/apps/AzureSqlConnector/IdentityQuery.cs new file mode 100644 index 0000000000..6c6cff84a1 --- /dev/null +++ b/doc/apps/AzureSqlConnector/IdentityQuery.cs @@ -0,0 +1,25 @@ +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + /// + /// Shared SQL text used by both (UI-thread variant) and + /// (worker-thread variant). Keeping the literal in one place + /// avoids drift when one variant gains a new column. + /// + internal static class IdentityQuery + { + public const string CommandText = + "SELECT " + + " SUSER_SNAME() AS LoggedInUser, " + + " ORIGINAL_LOGIN() AS OriginalLogin, " + + " USER_NAME() AS DatabaseUser, " + + " SUSER_ID() AS LoginSid, " + + " DB_NAME() AS DatabaseName, " + + " @@SERVERNAME AS ServerName, " + + " HOST_NAME() AS ClientHost, " + + " APP_NAME() AS AppName, " + + " SESSION_USER AS SessionUser, " + + " CURRENT_USER AS CurrentUser, " + + " @@SPID AS SessionId, " + + " @@VERSION AS ServerVersion;"; + } +} diff --git a/doc/apps/AzureSqlConnector/MainForm.Designer.cs b/doc/apps/AzureSqlConnector/MainForm.Designer.cs new file mode 100644 index 0000000000..4dd6c5a047 --- /dev/null +++ b/doc/apps/AzureSqlConnector/MainForm.Designer.cs @@ -0,0 +1,394 @@ +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + partial class MainForm + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + this.lblServer = new System.Windows.Forms.Label(); + this.txtServer = new System.Windows.Forms.TextBox(); + this.lblDatabase = new System.Windows.Forms.Label(); + this.txtDatabase = new System.Windows.Forms.TextBox(); + this.lblAuthentication = new System.Windows.Forms.Label(); + this.cmbAuthentication = new System.Windows.Forms.ComboBox(); + this.lblUserId = new System.Windows.Forms.Label(); + this.txtUserId = new System.Windows.Forms.TextBox(); + this.lblPassword = new System.Windows.Forms.Label(); + this.txtPassword = new System.Windows.Forms.TextBox(); + this.lblEncrypt = new System.Windows.Forms.Label(); + this.cmbEncrypt = new System.Windows.Forms.ComboBox(); + this.chkTrustServerCertificate = new System.Windows.Forms.CheckBox(); + this.lblTimeout = new System.Windows.Forms.Label(); + this.numTimeout = new System.Windows.Forms.NumericUpDown(); + this.lblOpenMode = new System.Windows.Forms.Label(); + this.cmbOpenMode = new System.Windows.Forms.ComboBox(); + this.lblConnectionString = new System.Windows.Forms.Label(); + this.txtConnectionString = new System.Windows.Forms.TextBox(); + this.btnBuild = new System.Windows.Forms.Button(); + this.btnTest = new System.Windows.Forms.Button(); + this.btnCopy = new System.Windows.Forms.Button(); + this.btnClear = new System.Windows.Forms.Button(); + this.btnWhoAmI = new System.Windows.Forms.Button(); + this.lblStatus = new System.Windows.Forms.Label(); + this.txtStatus = new System.Windows.Forms.TextBox(); + this.statusStrip = new System.Windows.Forms.StatusStrip(); + this.statusLabel = new System.Windows.Forms.ToolStripStatusLabel(); + ((System.ComponentModel.ISupportInitialize)(this.numTimeout)).BeginInit(); + this.statusStrip.SuspendLayout(); + this.SuspendLayout(); + // + // lblServer + // + this.lblServer.AutoSize = true; + this.lblServer.Location = new System.Drawing.Point(16, 18); + this.lblServer.Name = "lblServer"; + this.lblServer.Size = new System.Drawing.Size(75, 13); + this.lblServer.TabIndex = 0; + this.lblServer.Text = "&Server name:"; + // + // txtServer + // + this.txtServer.Location = new System.Drawing.Point(150, 15); + this.txtServer.Name = "txtServer"; + this.txtServer.Size = new System.Drawing.Size(400, 20); + this.txtServer.TabIndex = 1; + // + // lblDatabase + // + this.lblDatabase.AutoSize = true; + this.lblDatabase.Location = new System.Drawing.Point(16, 48); + this.lblDatabase.Name = "lblDatabase"; + this.lblDatabase.Size = new System.Drawing.Size(86, 13); + this.lblDatabase.TabIndex = 2; + this.lblDatabase.Text = "&Database name:"; + // + // txtDatabase + // + this.txtDatabase.Location = new System.Drawing.Point(150, 45); + this.txtDatabase.Name = "txtDatabase"; + this.txtDatabase.Size = new System.Drawing.Size(400, 20); + this.txtDatabase.TabIndex = 3; + // + // lblAuthentication + // + this.lblAuthentication.AutoSize = true; + this.lblAuthentication.Location = new System.Drawing.Point(16, 78); + this.lblAuthentication.Name = "lblAuthentication"; + this.lblAuthentication.Size = new System.Drawing.Size(80, 13); + this.lblAuthentication.TabIndex = 4; + this.lblAuthentication.Text = "&Authentication:"; + // + // cmbAuthentication + // + this.cmbAuthentication.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList; + this.cmbAuthentication.FormattingEnabled = true; + this.cmbAuthentication.Location = new System.Drawing.Point(150, 75); + this.cmbAuthentication.Name = "cmbAuthentication"; + this.cmbAuthentication.Size = new System.Drawing.Size(400, 21); + this.cmbAuthentication.TabIndex = 5; + this.cmbAuthentication.SelectedIndexChanged += new System.EventHandler(this.cmbAuthentication_SelectedIndexChanged); + // + // lblUserId + // + this.lblUserId.AutoSize = true; + this.lblUserId.Location = new System.Drawing.Point(16, 108); + this.lblUserId.Name = "lblUserId"; + this.lblUserId.Size = new System.Drawing.Size(45, 13); + this.lblUserId.TabIndex = 6; + this.lblUserId.Text = "&User ID:"; + // + // txtUserId + // + this.txtUserId.Location = new System.Drawing.Point(150, 105); + this.txtUserId.Name = "txtUserId"; + this.txtUserId.Size = new System.Drawing.Size(400, 20); + this.txtUserId.TabIndex = 7; + // + // lblPassword + // + this.lblPassword.AutoSize = true; + this.lblPassword.Location = new System.Drawing.Point(16, 138); + this.lblPassword.Name = "lblPassword"; + this.lblPassword.Size = new System.Drawing.Size(56, 13); + this.lblPassword.TabIndex = 8; + this.lblPassword.Text = "&Password:"; + // + // txtPassword + // + this.txtPassword.Location = new System.Drawing.Point(150, 135); + this.txtPassword.Name = "txtPassword"; + this.txtPassword.Size = new System.Drawing.Size(400, 20); + this.txtPassword.TabIndex = 9; + this.txtPassword.UseSystemPasswordChar = true; + // + // lblEncrypt + // + this.lblEncrypt.AutoSize = true; + this.lblEncrypt.Location = new System.Drawing.Point(16, 168); + this.lblEncrypt.Name = "lblEncrypt"; + this.lblEncrypt.Size = new System.Drawing.Size(46, 13); + this.lblEncrypt.TabIndex = 10; + this.lblEncrypt.Text = "&Encrypt:"; + // + // cmbEncrypt + // + this.cmbEncrypt.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList; + this.cmbEncrypt.FormattingEnabled = true; + this.cmbEncrypt.Location = new System.Drawing.Point(150, 165); + this.cmbEncrypt.Name = "cmbEncrypt"; + this.cmbEncrypt.Size = new System.Drawing.Size(200, 21); + this.cmbEncrypt.TabIndex = 11; + // + // chkTrustServerCertificate + // + this.chkTrustServerCertificate.AutoSize = true; + this.chkTrustServerCertificate.Location = new System.Drawing.Point(370, 167); + this.chkTrustServerCertificate.Name = "chkTrustServerCertificate"; + this.chkTrustServerCertificate.Size = new System.Drawing.Size(149, 17); + this.chkTrustServerCertificate.TabIndex = 12; + this.chkTrustServerCertificate.Text = "&Trust server certificate"; + this.chkTrustServerCertificate.UseVisualStyleBackColor = true; + // + // lblTimeout + // + this.lblTimeout.AutoSize = true; + this.lblTimeout.Location = new System.Drawing.Point(16, 198); + this.lblTimeout.Name = "lblTimeout"; + this.lblTimeout.Size = new System.Drawing.Size(101, 13); + this.lblTimeout.TabIndex = 13; + this.lblTimeout.Text = "Connect timeout (s):"; + // + // numTimeout + // + this.numTimeout.Location = new System.Drawing.Point(150, 196); + this.numTimeout.Maximum = new decimal(new int[] { 600, 0, 0, 0 }); + this.numTimeout.Minimum = new decimal(new int[] { 1, 0, 0, 0 }); + this.numTimeout.Name = "numTimeout"; + this.numTimeout.Size = new System.Drawing.Size(80, 20); + this.numTimeout.TabIndex = 14; + this.numTimeout.Value = new decimal(new int[] { 30, 0, 0, 0 }); + // + // lblOpenMode + // + this.lblOpenMode.AutoSize = true; + this.lblOpenMode.Location = new System.Drawing.Point(260, 198); + this.lblOpenMode.Name = "lblOpenMode"; + this.lblOpenMode.Size = new System.Drawing.Size(67, 13); + this.lblOpenMode.TabIndex = 25; + this.lblOpenMode.Text = "&Open mode:"; + // + // cmbOpenMode + // + this.cmbOpenMode.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList; + this.cmbOpenMode.FormattingEnabled = true; + this.cmbOpenMode.Location = new System.Drawing.Point(350, 195); + this.cmbOpenMode.Name = "cmbOpenMode"; + this.cmbOpenMode.Size = new System.Drawing.Size(200, 21); + this.cmbOpenMode.TabIndex = 26; + // + // lblConnectionString + // + this.lblConnectionString.AutoSize = true; + this.lblConnectionString.Location = new System.Drawing.Point(16, 230); + this.lblConnectionString.Name = "lblConnectionString"; + this.lblConnectionString.Size = new System.Drawing.Size(94, 13); + this.lblConnectionString.TabIndex = 15; + this.lblConnectionString.Text = "Connection string:"; + // + // txtConnectionString + // + this.txtConnectionString.Location = new System.Drawing.Point(16, 246); + this.txtConnectionString.Multiline = true; + this.txtConnectionString.Name = "txtConnectionString"; + this.txtConnectionString.ReadOnly = true; + this.txtConnectionString.ScrollBars = System.Windows.Forms.ScrollBars.Vertical; + this.txtConnectionString.Size = new System.Drawing.Size(534, 60); + this.txtConnectionString.TabIndex = 16; + this.txtConnectionString.BackColor = System.Drawing.SystemColors.Info; + // + // btnBuild + // + this.btnBuild.Location = new System.Drawing.Point(16, 316); + this.btnBuild.Name = "btnBuild"; + this.btnBuild.Size = new System.Drawing.Size(140, 26); + this.btnBuild.TabIndex = 17; + this.btnBuild.Text = "&Build Connection String"; + this.btnBuild.UseVisualStyleBackColor = true; + this.btnBuild.Click += new System.EventHandler(this.btnBuild_Click); + // + // btnTest + // + this.btnTest.Location = new System.Drawing.Point(166, 316); + this.btnTest.Name = "btnTest"; + this.btnTest.Size = new System.Drawing.Size(120, 26); + this.btnTest.TabIndex = 18; + this.btnTest.Text = "Te&st Connection"; + this.btnTest.UseVisualStyleBackColor = true; + this.btnTest.Click += new System.EventHandler(this.btnTest_Click); + // + // btnCopy + // + this.btnCopy.Location = new System.Drawing.Point(296, 316); + this.btnCopy.Name = "btnCopy"; + this.btnCopy.Size = new System.Drawing.Size(120, 26); + this.btnCopy.TabIndex = 19; + this.btnCopy.Text = "Cop&y to Clipboard"; + this.btnCopy.UseVisualStyleBackColor = true; + this.btnCopy.Click += new System.EventHandler(this.btnCopy_Click); + // + // btnClear + // + this.btnClear.Location = new System.Drawing.Point(426, 316); + this.btnClear.Name = "btnClear"; + this.btnClear.Size = new System.Drawing.Size(124, 26); + this.btnClear.TabIndex = 20; + this.btnClear.Text = "Cl&ear All"; + this.btnClear.UseVisualStyleBackColor = true; + this.btnClear.Click += new System.EventHandler(this.btnClear_Click); + // + // btnWhoAmI + // + this.btnWhoAmI.Location = new System.Drawing.Point(16, 348); + this.btnWhoAmI.Name = "btnWhoAmI"; + this.btnWhoAmI.Size = new System.Drawing.Size(534, 26); + this.btnWhoAmI.TabIndex = 21; + this.btnWhoAmI.Text = "&Who Am I? (run identity query on the database)"; + this.btnWhoAmI.UseVisualStyleBackColor = true; + this.btnWhoAmI.Click += new System.EventHandler(this.btnWhoAmI_Click); + // + // lblStatus + // + this.lblStatus.AutoSize = true; + this.lblStatus.Location = new System.Drawing.Point(16, 386); + this.lblStatus.Name = "lblStatus"; + this.lblStatus.Size = new System.Drawing.Size(40, 13); + this.lblStatus.TabIndex = 22; + this.lblStatus.Text = "Result:"; + // + // txtStatus + // + this.txtStatus.Location = new System.Drawing.Point(16, 402); + this.txtStatus.Multiline = true; + this.txtStatus.Name = "txtStatus"; + this.txtStatus.ReadOnly = true; + this.txtStatus.ScrollBars = System.Windows.Forms.ScrollBars.Both; + this.txtStatus.Size = new System.Drawing.Size(534, 160); + this.txtStatus.TabIndex = 23; + this.txtStatus.WordWrap = false; + this.txtStatus.Font = new System.Drawing.Font("Consolas", 9F); + // + // statusStrip + // + this.statusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { + this.statusLabel}); + this.statusStrip.Location = new System.Drawing.Point(0, 578); + this.statusStrip.Name = "statusStrip"; + this.statusStrip.Size = new System.Drawing.Size(566, 22); + this.statusStrip.TabIndex = 24; + // + // statusLabel + // + this.statusLabel.Name = "statusLabel"; + this.statusLabel.Size = new System.Drawing.Size(39, 17); + this.statusLabel.Text = "Ready"; + // + // MainForm + // + this.AcceptButton = this.btnTest; + this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); + this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + this.ClientSize = new System.Drawing.Size(566, 600); + this.Controls.Add(this.statusStrip); + this.Controls.Add(this.txtStatus); + this.Controls.Add(this.lblStatus); + this.Controls.Add(this.btnWhoAmI); + this.Controls.Add(this.btnClear); + this.Controls.Add(this.btnCopy); + this.Controls.Add(this.btnTest); + this.Controls.Add(this.btnBuild); + this.Controls.Add(this.txtConnectionString); + this.Controls.Add(this.lblConnectionString); + this.Controls.Add(this.cmbOpenMode); + this.Controls.Add(this.lblOpenMode); + this.Controls.Add(this.numTimeout); + this.Controls.Add(this.lblTimeout); + this.Controls.Add(this.chkTrustServerCertificate); + this.Controls.Add(this.cmbEncrypt); + this.Controls.Add(this.lblEncrypt); + this.Controls.Add(this.txtPassword); + this.Controls.Add(this.lblPassword); + this.Controls.Add(this.txtUserId); + this.Controls.Add(this.lblUserId); + this.Controls.Add(this.cmbAuthentication); + this.Controls.Add(this.lblAuthentication); + this.Controls.Add(this.txtDatabase); + this.Controls.Add(this.lblDatabase); + this.Controls.Add(this.txtServer); + this.Controls.Add(this.lblServer); + this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle; + this.MaximizeBox = false; + this.Name = "MainForm"; + this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen; + this.Text = "Azure SQL Connector"; + ((System.ComponentModel.ISupportInitialize)(this.numTimeout)).EndInit(); + this.statusStrip.ResumeLayout(false); + this.statusStrip.PerformLayout(); + this.ResumeLayout(false); + this.PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label lblServer; + private System.Windows.Forms.TextBox txtServer; + private System.Windows.Forms.Label lblDatabase; + private System.Windows.Forms.TextBox txtDatabase; + private System.Windows.Forms.Label lblAuthentication; + private System.Windows.Forms.ComboBox cmbAuthentication; + private System.Windows.Forms.Label lblUserId; + private System.Windows.Forms.TextBox txtUserId; + private System.Windows.Forms.Label lblPassword; + private System.Windows.Forms.TextBox txtPassword; + private System.Windows.Forms.Label lblEncrypt; + private System.Windows.Forms.ComboBox cmbEncrypt; + private System.Windows.Forms.CheckBox chkTrustServerCertificate; + private System.Windows.Forms.Label lblTimeout; + private System.Windows.Forms.NumericUpDown numTimeout; + private System.Windows.Forms.Label lblOpenMode; + private System.Windows.Forms.ComboBox cmbOpenMode; + private System.Windows.Forms.Label lblConnectionString; + private System.Windows.Forms.TextBox txtConnectionString; + private System.Windows.Forms.Button btnBuild; + private System.Windows.Forms.Button btnTest; + private System.Windows.Forms.Button btnCopy; + private System.Windows.Forms.Button btnClear; + private System.Windows.Forms.Button btnWhoAmI; + private System.Windows.Forms.Label lblStatus; + private System.Windows.Forms.TextBox txtStatus; + private System.Windows.Forms.StatusStrip statusStrip; + private System.Windows.Forms.ToolStripStatusLabel statusLabel; + } +} diff --git a/doc/apps/AzureSqlConnector/MainForm.cs b/doc/apps/AzureSqlConnector/MainForm.cs new file mode 100644 index 0000000000..9cc6d0648c --- /dev/null +++ b/doc/apps/AzureSqlConnector/MainForm.cs @@ -0,0 +1,590 @@ +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Windows.Forms; +using Microsoft.Data.SqlClient; +using Microsoft.Identity.Client; + +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + /// + /// "UI-thread" variant of the connector form. Opens the SQL connection via + /// on the UI thread; the WinForms + /// keeps the message pump alive while + /// the async I/O completes, so the form remains responsive and MSAL.NET's embedded sign-in + /// browser (for ActiveDirectoryInteractive) parents itself correctly. + /// + public partial class MainForm : Form + { + // ────────────────────────────────────────────────────────────────── + #region Construction + + public MainForm() + { + InitializeComponent(); + this.Text = "Azure SQL Connector — UI thread"; + PopulateAuthenticationMethods(); + PopulateEncryptOptions(); + PopulateOpenModes(); + UpdateCredentialFieldsAvailability(); + + // Force the underlying Win32 window to be created NOW (on the UI thread) so we can + // safely hand its HWND to MSAL later. Even in async mode, MSAL.NET may invoke the + // parent-window callback from a worker thread (e.g. when the driver blocks on a + // synchronous Open()), and touching Form.Handle from a non-UI thread throws + // InvalidOperationException ("Cross-thread operation not valid"). + _ownerHwnd = this.Handle; + + RegisterActiveDirectoryProvider(); + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region UI Initialization + + private void PopulateAuthenticationMethods() + { + foreach (SqlAuthenticationMethod method in Enum.GetValues(typeof(SqlAuthenticationMethod))) + { + cmbAuthentication.Items.Add(method); + } + + cmbAuthentication.SelectedItem = SqlAuthenticationMethod.SqlPassword; + } + + private void PopulateEncryptOptions() + { + cmbEncrypt.Items.Add(EncryptDisplay.Mandatory); + cmbEncrypt.Items.Add(EncryptDisplay.Optional); + cmbEncrypt.Items.Add(EncryptDisplay.Strict); + cmbEncrypt.SelectedIndex = 0; + } + + private void PopulateOpenModes() + { + cmbOpenMode.Items.Add(OpenModeDisplay.Async); + cmbOpenMode.Items.Add(OpenModeDisplay.Sync); + cmbOpenMode.SelectedIndex = 0; + } + + /// + /// Registers a single for every + /// Entra ID authentication method and gives it the form's captured HWND as the parent + /// window owner. Both callbacks intentionally use the HWND captured in the constructor + /// () rather than this.Handle, because MSAL.NET can invoke + /// them from a worker thread (e.g. when the driver blocks on a synchronous Open() + /// or when its internal continuations resume off-UI). + /// + private void RegisterActiveDirectoryProvider() + { + ActiveDirectoryAuthenticationProvider provider = new ActiveDirectoryAuthenticationProvider(); + IntPtr ownerHwnd = _ownerHwnd; + +#if NETFRAMEWORK + // .NET Framework: parent the embedded WebView via the legacy IWin32Window API. + provider.SetIWin32WindowFunc(() => new Win32WindowHandle(ownerHwnd)); +#endif + + // Modern API: works on both .NET Framework and .NET 8+, and is the one MSAL's WAM + // broker consults on Windows. + provider.SetParentActivityOrWindowFunc(() => ownerHwnd); + + // Without this, MSAL's default device-code callback writes the prompt to + // Console.WriteLine, which is invisible in a WinForms host — the connection + // appears to hang while MSAL polls for a code the user never sees. + provider.SetDeviceCodeFlowCallback(DeviceCodeFlowCallback); + + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryIntegrated, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryInteractive, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryServicePrincipal, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryDeviceCodeFlow, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryManagedIdentity, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryMSI, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryDefault, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity, provider); + #pragma warning disable CS0618 // Type or member is obsolete + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryPassword, provider); + #pragma warning restore CS0618 // Type or member is obsolete + } + + /// + /// Device Code Flow callback. MSAL invokes this on a worker thread before it begins + /// polling the token endpoint. We surface the user code three ways so the user always + /// sees it: (1) appended to the log textbox via BeginInvoke (works whenever the UI + /// thread is pumping — async OpenAsync), (2) the verification URL launched in + /// the default browser, and (3) a modal owned by the MSAL worker thread (works even + /// when the UI thread is blocked by a synchronous Open()). MSAL polling waits + /// for the returned Task to complete, so dismissing the dialog also resumes polling. + /// + private Task DeviceCodeFlowCallback(DeviceCodeResult result) + { + string message = result.Message; + string url = result.VerificationUrl; + string code = result.UserCode; + + if (IsHandleCreated) + { + try + { + BeginInvoke((Action)(() => + { + AppendStatus(string.Empty); + AppendStatus("=== Device Code Flow ==="); + AppendStatus(message); + })); + } + catch (InvalidOperationException) + { + // Form is closing or handle was destroyed; fall through to the modal. + } + } + + try + { + Process.Start(new ProcessStartInfo(url) { UseShellExecute = true }); + } + catch + { + // Best-effort; the modal below still shows the URL and code. + } + + MessageBox.Show( + "Sign in to complete Device Code Flow:" + Environment.NewLine + Environment.NewLine + + " URL : " + url + Environment.NewLine + + " Code: " + code + Environment.NewLine + Environment.NewLine + + "A browser window has been opened. Enter the code above, complete sign-in," + + Environment.NewLine + "then click OK to resume the connection.", + "Device Code Flow", + MessageBoxButtons.OK, + MessageBoxIcon.Information); + + return Task.CompletedTask; + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Event Handlers + + private void cmbAuthentication_SelectedIndexChanged(object sender, EventArgs e) + { + UpdateCredentialFieldsAvailability(); + } + + private void btnBuild_Click(object sender, EventArgs e) + { + try + { + SqlConnectionStringBuilder builder = BuildConnectionString(); + txtConnectionString.Text = MaskPassword(builder); + SetStatus("Connection string built successfully.", isError: false); + AppendStatus("Connection string built:\r\n" + MaskPassword(builder)); + } + catch (Exception ex) + { + txtConnectionString.Text = string.Empty; + SetStatus("Failed to build connection string.", isError: true); + AppendStatus("ERROR: " + ex.Message); + } + } + + private async void btnTest_Click(object sender, EventArgs e) + { + SqlConnectionStringBuilder builder; + try + { + builder = BuildConnectionString(); + txtConnectionString.Text = MaskPassword(builder); + } + catch (Exception ex) + { + SetStatus("Failed to build connection string.", isError: true); + AppendStatus("ERROR: " + ex.Message); + return; + } + + bool useAsync = IsAsyncOpenSelected(); + SetBusy(true, useAsync ? "Testing connection (OpenAsync)..." : "Testing connection (Open)..."); + AppendStatus(string.Empty); + AppendStatus("Testing connectivity to " + builder.DataSource + " (" + + (useAsync ? "OpenAsync" : "sync Open") + ") ..."); + + try + { + string serverVersion; + using (SqlConnection connection = new SqlConnection(builder.ConnectionString)) + { + await OpenConnectionAsync(connection, useAsync).ConfigureAwait(true); + serverVersion = connection.ServerVersion; + } + + SetStatus("Connected successfully.", isError: false); + AppendStatus("Connected successfully! Server version: " + serverVersion); + } + catch (SqlException ex) + { + SetStatus("Connection failed (SqlException).", isError: true); + AppendStatus("SqlException [" + ex.Number + "]: " + ex.Message + "\r\n" + ex.StackTrace); + } + catch (Exception ex) + { + SetStatus("Connection failed.", isError: true); + AppendStatus(ex.GetType().Name + ": " + ex.Message + "\r\n" + ex.StackTrace); + } + finally + { + SetBusy(false, null); + } + } + + private async void btnWhoAmI_Click(object sender, EventArgs e) + { + SqlConnectionStringBuilder builder; + try + { + builder = BuildConnectionString(); + txtConnectionString.Text = MaskPassword(builder); + } + catch (Exception ex) + { + SetStatus("Failed to build connection string.", isError: true); + AppendStatus("ERROR: " + ex.Message); + return; + } + + bool useAsync = IsAsyncOpenSelected(); + SetBusy(true, useAsync + ? "Querying logged-in identity (OpenAsync)..." + : "Querying logged-in identity (Open)..."); + AppendStatus(string.Empty); + AppendStatus("Running identity query against " + builder.DataSource + " (" + + (useAsync ? "OpenAsync" : "sync Open") + ") ..."); + + try + { + // Same UI-thread reasoning as btnTest_Click — keep the message pump alive for any + // ActiveDirectoryInteractive sign-in that may be required. + using (SqlConnection connection = new SqlConnection(builder.ConnectionString)) + { + await OpenConnectionAsync(connection, useAsync).ConfigureAwait(true); + + using (SqlCommand command = connection.CreateCommand()) + { + command.CommandText = IdentityQuery.CommandText; + + using (SqlDataReader reader = await command.ExecuteReaderAsync().ConfigureAwait(true)) + { + if (await reader.ReadAsync().ConfigureAwait(true)) + { + AppendStatus("Identity:"); + for (int i = 0; i < reader.FieldCount; i++) + { + string name = reader.GetName(i); + object value = reader.IsDBNull(i) ? "(null)" : reader.GetValue(i); + AppendStatus(" " + name.PadRight(16) + ": " + value); + } + SetStatus("Identity query succeeded.", isError: false); + } + else + { + SetStatus("Identity query returned no rows.", isError: true); + AppendStatus("(no rows returned)"); + } + } + } + } + } + catch (SqlException ex) + { + SetStatus("Identity query failed (SqlException).", isError: true); + AppendStatus("SqlException [" + ex.Number + "]: " + ex.Message); + } + catch (Exception ex) + { + SetStatus("Identity query failed.", isError: true); + AppendStatus(ex.GetType().Name + ": " + ex.Message); + } + finally + { + SetBusy(false, null); + } + } + + private void btnCopy_Click(object sender, EventArgs e) + { + if (string.IsNullOrEmpty(txtConnectionString.Text)) + { + SetStatus("Nothing to copy. Build the connection string first.", isError: true); + return; + } + + try + { + Clipboard.SetText(BuildConnectionString().ConnectionString); + SetStatus("Connection string copied to clipboard.", isError: false); + } + catch (Exception ex) + { + SetStatus("Failed to copy to clipboard.", isError: true); + AppendStatus("ERROR: " + ex.Message); + } + } + + private void btnClear_Click(object sender, EventArgs e) + { + txtServer.Clear(); + txtDatabase.Clear(); + txtUserId.Clear(); + txtPassword.Clear(); + txtConnectionString.Clear(); + txtStatus.Clear(); + cmbAuthentication.SelectedItem = SqlAuthenticationMethod.SqlPassword; + cmbEncrypt.SelectedIndex = 0; + cmbOpenMode.SelectedIndex = 0; + chkTrustServerCertificate.Checked = false; + numTimeout.Value = 30; + SetStatus("Ready", isError: false); + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Connection String Construction + + private SqlConnectionStringBuilder BuildConnectionString() + { + string server = (txtServer.Text ?? string.Empty).Trim(); + if (string.IsNullOrEmpty(server)) + { + throw new InvalidOperationException("Server name is required."); + } + + SqlAuthenticationMethod authMethod = (SqlAuthenticationMethod)cmbAuthentication.SelectedItem; + + SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder + { + DataSource = server, + ConnectTimeout = (int)numTimeout.Value, + }; + + string database = (txtDatabase.Text ?? string.Empty).Trim(); + if (!string.IsNullOrEmpty(database)) + { + builder.InitialCatalog = database; + } + + if (authMethod != SqlAuthenticationMethod.NotSpecified) + { + builder.Authentication = authMethod; + } + + if (RequiresUserAndPassword(authMethod)) + { + string userId = (txtUserId.Text ?? string.Empty).Trim(); + if (string.IsNullOrEmpty(userId)) + { + throw new InvalidOperationException( + "User ID is required for " + authMethod + " authentication."); + } + + builder.UserID = userId; + builder.Password = txtPassword.Text ?? string.Empty; + } + else if (authMethod == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal + || authMethod == SqlAuthenticationMethod.ActiveDirectoryManagedIdentity + || authMethod == SqlAuthenticationMethod.ActiveDirectoryMSI + || authMethod == SqlAuthenticationMethod.ActiveDirectoryInteractive + || authMethod == SqlAuthenticationMethod.ActiveDirectoryDeviceCodeFlow + || authMethod == SqlAuthenticationMethod.ActiveDirectoryDefault + || authMethod == SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity) + { + string userId = (txtUserId.Text ?? string.Empty).Trim(); + if (!string.IsNullOrEmpty(userId)) + { + builder.UserID = userId; + } + + if (authMethod == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal + && !string.IsNullOrEmpty(txtPassword.Text)) + { + builder.Password = txtPassword.Text; + } + } + + string encryptValue = cmbEncrypt.SelectedItem as string ?? EncryptDisplay.Mandatory; + switch (encryptValue) + { + case EncryptDisplay.Mandatory: + builder.Encrypt = SqlConnectionEncryptOption.Mandatory; + break; + case EncryptDisplay.Optional: + builder.Encrypt = SqlConnectionEncryptOption.Optional; + break; + case EncryptDisplay.Strict: + builder.Encrypt = SqlConnectionEncryptOption.Strict; + break; + } + + builder.TrustServerCertificate = chkTrustServerCertificate.Checked; + + return builder; + } + + private static bool RequiresUserAndPassword(SqlAuthenticationMethod method) + { + switch (method) + { + case SqlAuthenticationMethod.SqlPassword: +#pragma warning disable CS0618 // Type or member is obsolete + case SqlAuthenticationMethod.ActiveDirectoryPassword: +#pragma warning restore CS0618 + return true; + default: + return false; + } + } + + private static string MaskPassword(SqlConnectionStringBuilder builder) + { + if (string.IsNullOrEmpty(builder.Password)) + { + return builder.ConnectionString; + } + + SqlConnectionStringBuilder copy = new SqlConnectionStringBuilder(builder.ConnectionString) + { + Password = "********", + }; + return copy.ConnectionString; + } + + /// + /// Returns when the user picked Async (OpenAsync) in the + /// open-mode selector. Defaults to async if the selector has not been initialized yet. + /// + private bool IsAsyncOpenSelected() + { + return cmbOpenMode.SelectedItem as string != OpenModeDisplay.Sync; + } + + /// + /// Opens on the calling thread using either + /// or the synchronous + /// based on . The method itself is always async-returning so + /// callers can await uniformly; for the sync case it runs Open() inline on + /// the UI thread (which is supported with WAM broker because the broker dialog is hosted + /// by a separate process and does not need this thread's message pump). + /// + private static Task OpenConnectionAsync(SqlConnection connection, bool useAsync) + { + if (useAsync) + { + return connection.OpenAsync(); + } + + connection.Open(); + return Task.CompletedTask; + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region UI Helpers + + private void UpdateCredentialFieldsAvailability() + { + if (cmbAuthentication.SelectedItem == null) + { + return; + } + + SqlAuthenticationMethod method = (SqlAuthenticationMethod)cmbAuthentication.SelectedItem; + + bool userEnabled = method != SqlAuthenticationMethod.ActiveDirectoryIntegrated; + bool passwordEnabled = RequiresUserAndPassword(method) + || method == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal; + + txtUserId.Enabled = userEnabled; + txtPassword.Enabled = passwordEnabled; + + if (!passwordEnabled) + { + txtPassword.Clear(); + } + } + + private void SetStatus(string text, bool isError) + { + statusLabel.Text = text; + statusLabel.ForeColor = isError ? System.Drawing.Color.Firebrick : System.Drawing.Color.Black; + } + + private void AppendStatus(string line) + { + if (txtStatus.TextLength > 0) + { + txtStatus.AppendText(Environment.NewLine); + } + txtStatus.AppendText(line ?? string.Empty); + } + + private void SetBusy(bool busy, string statusText) + { + btnBuild.Enabled = !busy; + btnTest.Enabled = !busy; + btnCopy.Enabled = !busy; + btnClear.Enabled = !busy; + btnWhoAmI.Enabled = !busy; + cmbOpenMode.Enabled = !busy; + Cursor = busy ? Cursors.WaitCursor : Cursors.Default; + + if (statusText != null) + { + SetStatus(statusText, isError: false); + } + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Nested Types + + private static class EncryptDisplay + { + public const string Mandatory = "Mandatory"; + public const string Optional = "Optional"; + public const string Strict = "Strict"; + } + + private static class OpenModeDisplay + { + public const string Async = "Async (OpenAsync)"; + public const string Sync = "Sync (Open)"; + } + +#if NETFRAMEWORK + // Tiny IWin32Window wrapper around a raw HWND captured on the UI thread so MSAL.NET's + // legacy IWin32WindowFunc callback can safely return a window owner from a worker thread + // without ever touching Control.Handle off-UI. + private sealed class Win32WindowHandle : IWin32Window + { + private readonly IntPtr _hwnd; + public Win32WindowHandle(IntPtr hwnd) => _hwnd = hwnd; + public IntPtr Handle => _hwnd; + } +#endif + + #endregion + + // ─────────────────────────────────────────────────────────────── + #region Private Fields + + // The form's Win32 window handle, captured on the UI thread in the constructor. + // Read from worker threads by the Entra ID provider callbacks to parent MSAL's sign-in + // / WAM broker UI without illegally touching Control.Handle. + private readonly IntPtr _ownerHwnd; + + #endregion + } +} diff --git a/doc/apps/AzureSqlConnector/MainFormWorker.Designer.cs b/doc/apps/AzureSqlConnector/MainFormWorker.Designer.cs new file mode 100644 index 0000000000..c872ee38ab --- /dev/null +++ b/doc/apps/AzureSqlConnector/MainFormWorker.Designer.cs @@ -0,0 +1,383 @@ +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + partial class MainFormWorker + { + /// + /// Required designer variable. + /// + private System.ComponentModel.IContainer components = null; + + /// + /// Clean up any resources being used. + /// + /// true if managed resources should be disposed; otherwise, false. + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form Designer generated code + + /// + /// Required method for Designer support - do not modify + /// the contents of this method with the code editor. + /// + private void InitializeComponent() + { + this.lblServer = new System.Windows.Forms.Label(); + this.txtServer = new System.Windows.Forms.TextBox(); + this.lblDatabase = new System.Windows.Forms.Label(); + this.txtDatabase = new System.Windows.Forms.TextBox(); + this.lblAuthentication = new System.Windows.Forms.Label(); + this.cmbAuthentication = new System.Windows.Forms.ComboBox(); + this.lblUserId = new System.Windows.Forms.Label(); + this.txtUserId = new System.Windows.Forms.TextBox(); + this.lblPassword = new System.Windows.Forms.Label(); + this.txtPassword = new System.Windows.Forms.TextBox(); + this.lblEncrypt = new System.Windows.Forms.Label(); + this.cmbEncrypt = new System.Windows.Forms.ComboBox(); + this.chkTrustServerCertificate = new System.Windows.Forms.CheckBox(); + this.lblTimeout = new System.Windows.Forms.Label(); + this.numTimeout = new System.Windows.Forms.NumericUpDown(); + this.chkClearTokenCache = new System.Windows.Forms.CheckBox(); + this.lblConnectionString = new System.Windows.Forms.Label(); + this.txtConnectionString = new System.Windows.Forms.TextBox(); + this.btnBuild = new System.Windows.Forms.Button(); + this.btnTest = new System.Windows.Forms.Button(); + this.btnCopy = new System.Windows.Forms.Button(); + this.btnClear = new System.Windows.Forms.Button(); + this.btnWhoAmI = new System.Windows.Forms.Button(); + this.lblStatus = new System.Windows.Forms.Label(); + this.txtStatus = new System.Windows.Forms.TextBox(); + this.statusStrip = new System.Windows.Forms.StatusStrip(); + this.statusLabel = new System.Windows.Forms.ToolStripStatusLabel(); + ((System.ComponentModel.ISupportInitialize)(this.numTimeout)).BeginInit(); + this.statusStrip.SuspendLayout(); + this.SuspendLayout(); + // + // lblServer + // + this.lblServer.AutoSize = true; + this.lblServer.Location = new System.Drawing.Point(16, 18); + this.lblServer.Name = "lblServer"; + this.lblServer.Size = new System.Drawing.Size(75, 13); + this.lblServer.TabIndex = 0; + this.lblServer.Text = "&Server name:"; + // + // txtServer + // + this.txtServer.Location = new System.Drawing.Point(150, 15); + this.txtServer.Name = "txtServer"; + this.txtServer.Size = new System.Drawing.Size(400, 20); + this.txtServer.TabIndex = 1; + // + // lblDatabase + // + this.lblDatabase.AutoSize = true; + this.lblDatabase.Location = new System.Drawing.Point(16, 48); + this.lblDatabase.Name = "lblDatabase"; + this.lblDatabase.Size = new System.Drawing.Size(86, 13); + this.lblDatabase.TabIndex = 2; + this.lblDatabase.Text = "&Database name:"; + // + // txtDatabase + // + this.txtDatabase.Location = new System.Drawing.Point(150, 45); + this.txtDatabase.Name = "txtDatabase"; + this.txtDatabase.Size = new System.Drawing.Size(400, 20); + this.txtDatabase.TabIndex = 3; + // + // lblAuthentication + // + this.lblAuthentication.AutoSize = true; + this.lblAuthentication.Location = new System.Drawing.Point(16, 78); + this.lblAuthentication.Name = "lblAuthentication"; + this.lblAuthentication.Size = new System.Drawing.Size(80, 13); + this.lblAuthentication.TabIndex = 4; + this.lblAuthentication.Text = "&Authentication:"; + // + // cmbAuthentication + // + this.cmbAuthentication.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList; + this.cmbAuthentication.FormattingEnabled = true; + this.cmbAuthentication.Location = new System.Drawing.Point(150, 75); + this.cmbAuthentication.Name = "cmbAuthentication"; + this.cmbAuthentication.Size = new System.Drawing.Size(400, 21); + this.cmbAuthentication.TabIndex = 5; + this.cmbAuthentication.SelectedIndexChanged += new System.EventHandler(this.cmbAuthentication_SelectedIndexChanged); + // + // lblUserId + // + this.lblUserId.AutoSize = true; + this.lblUserId.Location = new System.Drawing.Point(16, 108); + this.lblUserId.Name = "lblUserId"; + this.lblUserId.Size = new System.Drawing.Size(45, 13); + this.lblUserId.TabIndex = 6; + this.lblUserId.Text = "&User ID:"; + // + // txtUserId + // + this.txtUserId.Location = new System.Drawing.Point(150, 105); + this.txtUserId.Name = "txtUserId"; + this.txtUserId.Size = new System.Drawing.Size(400, 20); + this.txtUserId.TabIndex = 7; + // + // lblPassword + // + this.lblPassword.AutoSize = true; + this.lblPassword.Location = new System.Drawing.Point(16, 138); + this.lblPassword.Name = "lblPassword"; + this.lblPassword.Size = new System.Drawing.Size(56, 13); + this.lblPassword.TabIndex = 8; + this.lblPassword.Text = "&Password:"; + // + // txtPassword + // + this.txtPassword.Location = new System.Drawing.Point(150, 135); + this.txtPassword.Name = "txtPassword"; + this.txtPassword.Size = new System.Drawing.Size(400, 20); + this.txtPassword.TabIndex = 9; + this.txtPassword.UseSystemPasswordChar = true; + // + // lblEncrypt + // + this.lblEncrypt.AutoSize = true; + this.lblEncrypt.Location = new System.Drawing.Point(16, 168); + this.lblEncrypt.Name = "lblEncrypt"; + this.lblEncrypt.Size = new System.Drawing.Size(46, 13); + this.lblEncrypt.TabIndex = 10; + this.lblEncrypt.Text = "&Encrypt:"; + // + // cmbEncrypt + // + this.cmbEncrypt.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList; + this.cmbEncrypt.FormattingEnabled = true; + this.cmbEncrypt.Location = new System.Drawing.Point(150, 165); + this.cmbEncrypt.Name = "cmbEncrypt"; + this.cmbEncrypt.Size = new System.Drawing.Size(200, 21); + this.cmbEncrypt.TabIndex = 11; + // + // chkTrustServerCertificate + // + this.chkTrustServerCertificate.AutoSize = true; + this.chkTrustServerCertificate.Location = new System.Drawing.Point(370, 167); + this.chkTrustServerCertificate.Name = "chkTrustServerCertificate"; + this.chkTrustServerCertificate.Size = new System.Drawing.Size(149, 17); + this.chkTrustServerCertificate.TabIndex = 12; + this.chkTrustServerCertificate.Text = "&Trust server certificate"; + this.chkTrustServerCertificate.UseVisualStyleBackColor = true; + // + // lblTimeout + // + this.lblTimeout.AutoSize = true; + this.lblTimeout.Location = new System.Drawing.Point(16, 198); + this.lblTimeout.Name = "lblTimeout"; + this.lblTimeout.Size = new System.Drawing.Size(101, 13); + this.lblTimeout.TabIndex = 13; + this.lblTimeout.Text = "Connect timeout (s):"; + // + // numTimeout + // + this.numTimeout.Location = new System.Drawing.Point(150, 196); + this.numTimeout.Maximum = new decimal(new int[] { 600, 0, 0, 0 }); + this.numTimeout.Minimum = new decimal(new int[] { 1, 0, 0, 0 }); + this.numTimeout.Name = "numTimeout"; + this.numTimeout.Size = new System.Drawing.Size(80, 20); + this.numTimeout.TabIndex = 14; + this.numTimeout.Value = new decimal(new int[] { 30, 0, 0, 0 }); + // + // chkClearTokenCache + // + this.chkClearTokenCache.AutoSize = true; + this.chkClearTokenCache.Location = new System.Drawing.Point(260, 198); + this.chkClearTokenCache.Name = "chkClearTokenCache"; + this.chkClearTokenCache.Size = new System.Drawing.Size(290, 17); + this.chkClearTokenCache.TabIndex = 15; + this.chkClearTokenCache.Text = "Clear MSAL token &cache before connect (force prompt)"; + this.chkClearTokenCache.UseVisualStyleBackColor = true; + // + // lblConnectionString + // + this.lblConnectionString.AutoSize = true; + this.lblConnectionString.Location = new System.Drawing.Point(16, 230); + this.lblConnectionString.Name = "lblConnectionString"; + this.lblConnectionString.Size = new System.Drawing.Size(94, 13); + this.lblConnectionString.TabIndex = 15; + this.lblConnectionString.Text = "Connection string:"; + // + // txtConnectionString + // + this.txtConnectionString.Location = new System.Drawing.Point(16, 246); + this.txtConnectionString.Multiline = true; + this.txtConnectionString.Name = "txtConnectionString"; + this.txtConnectionString.ReadOnly = true; + this.txtConnectionString.ScrollBars = System.Windows.Forms.ScrollBars.Vertical; + this.txtConnectionString.Size = new System.Drawing.Size(534, 60); + this.txtConnectionString.TabIndex = 16; + this.txtConnectionString.BackColor = System.Drawing.SystemColors.Info; + // + // btnBuild + // + this.btnBuild.Location = new System.Drawing.Point(16, 316); + this.btnBuild.Name = "btnBuild"; + this.btnBuild.Size = new System.Drawing.Size(140, 26); + this.btnBuild.TabIndex = 17; + this.btnBuild.Text = "&Build Connection String"; + this.btnBuild.UseVisualStyleBackColor = true; + this.btnBuild.Click += new System.EventHandler(this.btnBuild_Click); + // + // btnTest + // + this.btnTest.Location = new System.Drawing.Point(166, 316); + this.btnTest.Name = "btnTest"; + this.btnTest.Size = new System.Drawing.Size(120, 26); + this.btnTest.TabIndex = 18; + this.btnTest.Text = "Te&st Connection"; + this.btnTest.UseVisualStyleBackColor = true; + this.btnTest.Click += new System.EventHandler(this.btnTest_Click); + // + // btnCopy + // + this.btnCopy.Location = new System.Drawing.Point(296, 316); + this.btnCopy.Name = "btnCopy"; + this.btnCopy.Size = new System.Drawing.Size(120, 26); + this.btnCopy.TabIndex = 19; + this.btnCopy.Text = "Cop&y to Clipboard"; + this.btnCopy.UseVisualStyleBackColor = true; + this.btnCopy.Click += new System.EventHandler(this.btnCopy_Click); + // + // btnClear + // + this.btnClear.Location = new System.Drawing.Point(426, 316); + this.btnClear.Name = "btnClear"; + this.btnClear.Size = new System.Drawing.Size(124, 26); + this.btnClear.TabIndex = 20; + this.btnClear.Text = "Cl&ear All"; + this.btnClear.UseVisualStyleBackColor = true; + this.btnClear.Click += new System.EventHandler(this.btnClear_Click); + // + // btnWhoAmI + // + this.btnWhoAmI.Location = new System.Drawing.Point(16, 348); + this.btnWhoAmI.Name = "btnWhoAmI"; + this.btnWhoAmI.Size = new System.Drawing.Size(534, 26); + this.btnWhoAmI.TabIndex = 21; + this.btnWhoAmI.Text = "&Who Am I? (run identity query on the database)"; + this.btnWhoAmI.UseVisualStyleBackColor = true; + this.btnWhoAmI.Click += new System.EventHandler(this.btnWhoAmI_Click); + // + // lblStatus + // + this.lblStatus.AutoSize = true; + this.lblStatus.Location = new System.Drawing.Point(16, 386); + this.lblStatus.Name = "lblStatus"; + this.lblStatus.Size = new System.Drawing.Size(40, 13); + this.lblStatus.TabIndex = 22; + this.lblStatus.Text = "Result:"; + // + // txtStatus + // + this.txtStatus.Location = new System.Drawing.Point(16, 402); + this.txtStatus.Multiline = true; + this.txtStatus.Name = "txtStatus"; + this.txtStatus.ReadOnly = true; + this.txtStatus.ScrollBars = System.Windows.Forms.ScrollBars.Both; + this.txtStatus.Size = new System.Drawing.Size(534, 160); + this.txtStatus.TabIndex = 23; + this.txtStatus.WordWrap = false; + this.txtStatus.Font = new System.Drawing.Font("Consolas", 9F); + // + // statusStrip + // + this.statusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { + this.statusLabel}); + this.statusStrip.Location = new System.Drawing.Point(0, 578); + this.statusStrip.Name = "statusStrip"; + this.statusStrip.Size = new System.Drawing.Size(566, 22); + this.statusStrip.TabIndex = 24; + // + // statusLabel + // + this.statusLabel.Name = "statusLabel"; + this.statusLabel.Size = new System.Drawing.Size(39, 17); + this.statusLabel.Text = "Ready"; + // + // MainForm + // + this.AcceptButton = this.btnTest; + this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); + this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + this.ClientSize = new System.Drawing.Size(566, 600); + this.Controls.Add(this.statusStrip); + this.Controls.Add(this.txtStatus); + this.Controls.Add(this.lblStatus); + this.Controls.Add(this.btnWhoAmI); + this.Controls.Add(this.btnClear); + this.Controls.Add(this.btnCopy); + this.Controls.Add(this.btnTest); + this.Controls.Add(this.btnBuild); + this.Controls.Add(this.txtConnectionString); + this.Controls.Add(this.lblConnectionString); + this.Controls.Add(this.numTimeout); + this.Controls.Add(this.lblTimeout); + this.Controls.Add(this.chkClearTokenCache); + this.Controls.Add(this.chkTrustServerCertificate); + this.Controls.Add(this.cmbEncrypt); + this.Controls.Add(this.lblEncrypt); + this.Controls.Add(this.txtPassword); + this.Controls.Add(this.lblPassword); + this.Controls.Add(this.txtUserId); + this.Controls.Add(this.lblUserId); + this.Controls.Add(this.cmbAuthentication); + this.Controls.Add(this.lblAuthentication); + this.Controls.Add(this.txtDatabase); + this.Controls.Add(this.lblDatabase); + this.Controls.Add(this.txtServer); + this.Controls.Add(this.lblServer); + this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle; + this.MaximizeBox = false; + this.Name = "MainFormWorker"; + this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen; + this.Text = "Azure SQL Connector — Worker thread (Task.Run + Open)"; + ((System.ComponentModel.ISupportInitialize)(this.numTimeout)).EndInit(); + this.statusStrip.ResumeLayout(false); + this.statusStrip.PerformLayout(); + this.ResumeLayout(false); + this.PerformLayout(); + } + + #endregion + + private System.Windows.Forms.Label lblServer; + private System.Windows.Forms.TextBox txtServer; + private System.Windows.Forms.Label lblDatabase; + private System.Windows.Forms.TextBox txtDatabase; + private System.Windows.Forms.Label lblAuthentication; + private System.Windows.Forms.ComboBox cmbAuthentication; + private System.Windows.Forms.Label lblUserId; + private System.Windows.Forms.TextBox txtUserId; + private System.Windows.Forms.Label lblPassword; + private System.Windows.Forms.TextBox txtPassword; + private System.Windows.Forms.Label lblEncrypt; + private System.Windows.Forms.ComboBox cmbEncrypt; + private System.Windows.Forms.CheckBox chkTrustServerCertificate; + private System.Windows.Forms.Label lblTimeout; + private System.Windows.Forms.NumericUpDown numTimeout; + private System.Windows.Forms.CheckBox chkClearTokenCache; + private System.Windows.Forms.Label lblConnectionString; + private System.Windows.Forms.TextBox txtConnectionString; + private System.Windows.Forms.Button btnBuild; + private System.Windows.Forms.Button btnTest; + private System.Windows.Forms.Button btnCopy; + private System.Windows.Forms.Button btnClear; + private System.Windows.Forms.Button btnWhoAmI; + private System.Windows.Forms.Label lblStatus; + private System.Windows.Forms.TextBox txtStatus; + private System.Windows.Forms.StatusStrip statusStrip; + private System.Windows.Forms.ToolStripStatusLabel statusLabel; + } +} diff --git a/doc/apps/AzureSqlConnector/MainFormWorker.cs b/doc/apps/AzureSqlConnector/MainFormWorker.cs new file mode 100644 index 0000000000..8d344cf6c4 --- /dev/null +++ b/doc/apps/AzureSqlConnector/MainFormWorker.cs @@ -0,0 +1,598 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Windows.Forms; +using Microsoft.Data.SqlClient; +using Microsoft.Identity.Client; + +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + /// + /// "Worker thread" variant of the connector form. Opens the SQL connection synchronously + /// inside a call so the UI thread never blocks. + /// + /// + /// + /// The form's HWND is captured on the UI thread in the constructor and stashed in + /// . Both Entra ID parent-window callbacks return that captured + /// handle, so they are safe to invoke from the worker thread (touching Form.Handle + /// from a non-UI thread is illegal). + /// + /// + /// Compare with , which keeps Open on the UI thread and relies on + /// for responsiveness. + /// + /// + public partial class MainFormWorker : Form + { + // ────────────────────────────────────────────────────────────────── + #region Construction + + public MainFormWorker() + { + InitializeComponent(); + PopulateAuthenticationMethods(); + PopulateEncryptOptions(); + UpdateCredentialFieldsAvailability(); + + // Force the underlying Win32 window to be created NOW (on the UI thread) so we can + // safely capture its HWND for MSAL to use later from a worker thread. Touching + // Form.Handle from a non-UI thread is illegal, so we read it here once and stash it. + _ownerHwnd = this.Handle; + + RegisterActiveDirectoryProvider(); + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region UI Initialization + + private void PopulateAuthenticationMethods() + { + foreach (SqlAuthenticationMethod method in Enum.GetValues(typeof(SqlAuthenticationMethod))) + { + cmbAuthentication.Items.Add(method); + } + + cmbAuthentication.SelectedItem = SqlAuthenticationMethod.SqlPassword; + } + + private void PopulateEncryptOptions() + { + cmbEncrypt.Items.Add(EncryptDisplay.Mandatory); + cmbEncrypt.Items.Add(EncryptDisplay.Optional); + cmbEncrypt.Items.Add(EncryptDisplay.Strict); + cmbEncrypt.SelectedIndex = 0; + } + + /// + /// Registers a single for every + /// Entra ID authentication method and gives it the form's captured HWND as the parent + /// window owner. Both callbacks intentionally use the HWND captured in the constructor + /// () rather than this.Handle; they are invoked by MSAL on + /// the worker thread that called . + /// + private void RegisterActiveDirectoryProvider() + { + ActiveDirectoryAuthenticationProvider provider = new ActiveDirectoryAuthenticationProvider(); + IntPtr ownerHwnd = _ownerHwnd; + +#if NETFRAMEWORK + // .NET Framework: parent the embedded WebView via the legacy IWin32Window API. + provider.SetIWin32WindowFunc(() => new Win32WindowHandle(ownerHwnd)); +#endif + + // Modern API: works on both .NET Framework and .NET 8+, and is the one MSAL's WAM + // broker consults on Windows. + provider.SetParentActivityOrWindowFunc(() => ownerHwnd); + + // Without this, MSAL's default device-code callback writes the prompt to + // Console.WriteLine, which is invisible in a WinForms host — the connection + // appears to hang while MSAL polls for a code the user never sees. + provider.SetDeviceCodeFlowCallback(DeviceCodeFlowCallback); + + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryIntegrated, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryInteractive, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryServicePrincipal, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryDeviceCodeFlow, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryManagedIdentity, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryMSI, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryDefault, provider); + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity, provider); + #pragma warning disable CS0618 // Type or member is obsolete + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryPassword, provider); + #pragma warning restore CS0618 // Type or member is obsolete + } + + /// + /// Device Code Flow callback. MSAL invokes this on a worker thread before it begins + /// polling the token endpoint. We surface the user code three ways so the user always + /// sees it: (1) appended to the log textbox via BeginInvoke (the UI thread is free in + /// this variant because Open() runs on a Task.Run worker), (2) the verification URL + /// launched in the default browser, and (3) a modal owned by the MSAL worker thread. + /// MSAL polling waits for the returned Task to complete, so dismissing the dialog + /// also resumes polling. + /// + private Task DeviceCodeFlowCallback(DeviceCodeResult result) + { + string message = result.Message; + string url = result.VerificationUrl; + string code = result.UserCode; + + if (IsHandleCreated) + { + try + { + BeginInvoke((Action)(() => + { + AppendStatus(string.Empty); + AppendStatus("=== Device Code Flow ==="); + AppendStatus(message); + })); + } + catch (InvalidOperationException) + { + // Form is closing or handle was destroyed; fall through to the modal. + } + } + + try + { + Process.Start(new ProcessStartInfo(url) { UseShellExecute = true }); + } + catch + { + // Best-effort; the modal below still shows the URL and code. + } + + MessageBox.Show( + "Sign in to complete Device Code Flow:" + Environment.NewLine + Environment.NewLine + + " URL : " + url + Environment.NewLine + + " Code: " + code + Environment.NewLine + Environment.NewLine + + "A browser window has been opened. Enter the code above, complete sign-in," + + Environment.NewLine + "then click OK to resume the connection.", + "Device Code Flow", + MessageBoxButtons.OK, + MessageBoxIcon.Information); + + return Task.CompletedTask; + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Event Handlers + + private void cmbAuthentication_SelectedIndexChanged(object sender, EventArgs e) + { + UpdateCredentialFieldsAvailability(); + } + + private void btnBuild_Click(object sender, EventArgs e) + { + try + { + SqlConnectionStringBuilder builder = BuildConnectionString(); + txtConnectionString.Text = MaskPassword(builder); + SetStatus("Connection string built successfully.", isError: false); + AppendStatus("Connection string built:\r\n" + MaskPassword(builder)); + } + catch (Exception ex) + { + txtConnectionString.Text = string.Empty; + SetStatus("Failed to build connection string.", isError: true); + AppendStatus("ERROR: " + ex.Message); + } + } + + private async void btnTest_Click(object sender, EventArgs e) + { + SqlConnectionStringBuilder builder; + try + { + builder = BuildConnectionString(); + txtConnectionString.Text = MaskPassword(builder); + } + catch (Exception ex) + { + SetStatus("Failed to build connection string.", isError: true); + AppendStatus("ERROR: " + ex.Message); + return; + } + + SetBusy(true, "Testing connection..."); + AppendStatus(string.Empty); + AppendStatus("Testing connectivity to " + builder.DataSource + " ..."); + + MaybeClearTokenCache(); + + try + { + // Run Open() on a thread-pool worker so the UI thread never blocks. The await + // continuation hops back onto the UI thread automatically (the awaiter captures + // the current SynchronizationContext), so it is safe to touch the form's controls + // after the await. + // + // The Entra ID interactive / WAM flows still find a parent window because we + // captured the form's HWND on the UI thread in the constructor and the callbacks + // registered in RegisterActiveDirectoryProvider return that captured handle (no + // UI-thread-only Form.Handle access from the worker thread). + string connectionString = builder.ConnectionString; + string serverVersion = await Task.Run(() => + { + using (SqlConnection connection = new SqlConnection(connectionString)) + { + connection.Open(); + return connection.ServerVersion; + } + }).ConfigureAwait(true); + + SetStatus("Connected successfully.", isError: false); + AppendStatus("Connected successfully! Server version: " + serverVersion); + } + catch (SqlException ex) + { + SetStatus("Connection failed (SqlException).", isError: true); + AppendStatus("SqlException [" + ex.Number + "]: " + ex.Message); + } + catch (Exception ex) + { + SetStatus("Connection failed.", isError: true); + AppendStatus(ex.GetType().Name + ": " + ex.Message); + } + finally + { + SetBusy(false, null); + } + } + + private async void btnWhoAmI_Click(object sender, EventArgs e) + { + SqlConnectionStringBuilder builder; + try + { + builder = BuildConnectionString(); + txtConnectionString.Text = MaskPassword(builder); + } + catch (Exception ex) + { + SetStatus("Failed to build connection string.", isError: true); + AppendStatus("ERROR: " + ex.Message); + return; + } + + SetBusy(true, "Querying logged-in identity..."); + AppendStatus(string.Empty); + AppendStatus("Running identity query against " + builder.DataSource + " ..."); + + MaybeClearTokenCache(); + + try + { + // Run the whole open + query + read on a worker thread so the UI never blocks. + // We materialize the single result row into a List<(name, value)> on the worker + // and then format it on the UI thread once the await returns. + string connectionString = builder.ConnectionString; + List<(string Name, object Value)> row = await Task.Run(() => + { + using (SqlConnection connection = new SqlConnection(connectionString)) + { + connection.Open(); + + using (SqlCommand command = connection.CreateCommand()) + { + command.CommandText = IdentityQuery.CommandText; + + using (SqlDataReader reader = command.ExecuteReader()) + { + if (!reader.Read()) + { + return null; + } + + var fields = new List<(string, object)>(reader.FieldCount); + for (int i = 0; i < reader.FieldCount; i++) + { + object value = reader.IsDBNull(i) ? "(null)" : reader.GetValue(i); + fields.Add((reader.GetName(i), value)); + } + return fields; + } + } + } + }).ConfigureAwait(true); + + if (row is null) + { + SetStatus("Identity query returned no rows.", isError: true); + AppendStatus("(no rows returned)"); + } + else + { + AppendStatus("Identity:"); + foreach (var (name, value) in row) + { + AppendStatus(" " + name.PadRight(16) + ": " + value); + } + SetStatus("Identity query succeeded.", isError: false); + } + } + catch (SqlException ex) + { + SetStatus("Identity query failed (SqlException).", isError: true); + AppendStatus("SqlException [" + ex.Number + "]: " + ex.Message); + } + catch (Exception ex) + { + SetStatus("Identity query failed.", isError: true); + AppendStatus(ex.GetType().Name + ": " + ex.Message); + } + finally + { + SetBusy(false, null); + } + } + + private void btnCopy_Click(object sender, EventArgs e) + { + if (string.IsNullOrEmpty(txtConnectionString.Text)) + { + SetStatus("Nothing to copy. Build the connection string first.", isError: true); + return; + } + + try + { + Clipboard.SetText(BuildConnectionString().ConnectionString); + SetStatus("Connection string copied to clipboard.", isError: false); + } + catch (Exception ex) + { + SetStatus("Failed to copy to clipboard.", isError: true); + AppendStatus("ERROR: " + ex.Message); + } + } + + private void btnClear_Click(object sender, EventArgs e) + { + txtServer.Clear(); + txtDatabase.Clear(); + txtUserId.Clear(); + txtPassword.Clear(); + txtConnectionString.Clear(); + txtStatus.Clear(); + cmbAuthentication.SelectedItem = SqlAuthenticationMethod.SqlPassword; + cmbEncrypt.SelectedIndex = 0; + chkTrustServerCertificate.Checked = false; + numTimeout.Value = 30; + SetStatus("Ready", isError: false); + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Connection String Construction + + private SqlConnectionStringBuilder BuildConnectionString() + { + string server = (txtServer.Text ?? string.Empty).Trim(); + if (string.IsNullOrEmpty(server)) + { + throw new InvalidOperationException("Server name is required."); + } + + SqlAuthenticationMethod authMethod = (SqlAuthenticationMethod)cmbAuthentication.SelectedItem; + + SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder + { + DataSource = server, + ConnectTimeout = (int)numTimeout.Value, + }; + + string database = (txtDatabase.Text ?? string.Empty).Trim(); + if (!string.IsNullOrEmpty(database)) + { + builder.InitialCatalog = database; + } + + if (authMethod != SqlAuthenticationMethod.NotSpecified) + { + builder.Authentication = authMethod; + } + + if (RequiresUserAndPassword(authMethod)) + { + string userId = (txtUserId.Text ?? string.Empty).Trim(); + if (string.IsNullOrEmpty(userId)) + { + throw new InvalidOperationException( + "User ID is required for " + authMethod + " authentication."); + } + + builder.UserID = userId; + builder.Password = txtPassword.Text ?? string.Empty; + } + else if (authMethod == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal + || authMethod == SqlAuthenticationMethod.ActiveDirectoryManagedIdentity + || authMethod == SqlAuthenticationMethod.ActiveDirectoryMSI + || authMethod == SqlAuthenticationMethod.ActiveDirectoryInteractive + || authMethod == SqlAuthenticationMethod.ActiveDirectoryDeviceCodeFlow + || authMethod == SqlAuthenticationMethod.ActiveDirectoryDefault + || authMethod == SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity) + { + string userId = (txtUserId.Text ?? string.Empty).Trim(); + if (!string.IsNullOrEmpty(userId)) + { + builder.UserID = userId; + } + + if (authMethod == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal + && !string.IsNullOrEmpty(txtPassword.Text)) + { + builder.Password = txtPassword.Text; + } + } + + string encryptValue = cmbEncrypt.SelectedItem as string ?? EncryptDisplay.Mandatory; + switch (encryptValue) + { + case EncryptDisplay.Mandatory: + builder.Encrypt = SqlConnectionEncryptOption.Mandatory; + break; + case EncryptDisplay.Optional: + builder.Encrypt = SqlConnectionEncryptOption.Optional; + break; + case EncryptDisplay.Strict: + builder.Encrypt = SqlConnectionEncryptOption.Strict; + break; + } + + builder.TrustServerCertificate = chkTrustServerCertificate.Checked; + + return builder; + } + + private static bool RequiresUserAndPassword(SqlAuthenticationMethod method) + { + switch (method) + { + case SqlAuthenticationMethod.SqlPassword: +#pragma warning disable CS0618 // Type or member is obsolete + case SqlAuthenticationMethod.ActiveDirectoryPassword: +#pragma warning restore CS0618 + return true; + default: + return false; + } + } + + private static string MaskPassword(SqlConnectionStringBuilder builder) + { + if (string.IsNullOrEmpty(builder.Password)) + { + return builder.ConnectionString; + } + + SqlConnectionStringBuilder copy = new SqlConnectionStringBuilder(builder.ConnectionString) + { + Password = "********", + }; + return copy.ConnectionString; + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region UI Helpers + + private void UpdateCredentialFieldsAvailability() + { + if (cmbAuthentication.SelectedItem == null) + { + return; + } + + SqlAuthenticationMethod method = (SqlAuthenticationMethod)cmbAuthentication.SelectedItem; + + bool userEnabled = method != SqlAuthenticationMethod.ActiveDirectoryIntegrated; + bool passwordEnabled = RequiresUserAndPassword(method) + || method == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal; + + txtUserId.Enabled = userEnabled; + txtPassword.Enabled = passwordEnabled; + + if (!passwordEnabled) + { + txtPassword.Clear(); + } + } + + private void SetStatus(string text, bool isError) + { + statusLabel.Text = text; + statusLabel.ForeColor = isError ? System.Drawing.Color.Firebrick : System.Drawing.Color.Black; + } + + private void AppendStatus(string line) + { + if (txtStatus.TextLength > 0) + { + txtStatus.AppendText(Environment.NewLine); + } + txtStatus.AppendText(line ?? string.Empty); + } + + private void SetBusy(bool busy, string statusText) + { + btnBuild.Enabled = !busy; + btnTest.Enabled = !busy; + btnCopy.Enabled = !busy; + btnClear.Enabled = !busy; + btnWhoAmI.Enabled = !busy; + Cursor = busy ? Cursors.WaitCursor : Cursors.Default; + + if (statusText != null) + { + SetStatus(statusText, isError: false); + } + } + + // Only drops the in-process PCA / TokenCredential maps; MSAL's persistent on-disk cache + // and WAM broker accounts are untouched. Sufficient to demo a worker-thread interactive + // prompt when the persistent cache has already been cleared (fresh run or no WAM account + // bound), and a useful reset between back-to-back connects within a single session. + private void MaybeClearTokenCache() + { + if (!chkClearTokenCache.Checked) + { + return; + } + + ActiveDirectoryAuthenticationProvider.ClearUserTokenCache(); + AppendStatus("Cleared in-process MSAL token cache."); + } + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Nested Types + + private static class EncryptDisplay + { + public const string Mandatory = "Mandatory"; + public const string Optional = "Optional"; + public const string Strict = "Strict"; + } + + /// + /// Tiny wrapper around a raw HWND captured on the UI thread. + /// Used so that MSAL.NET's IWin32WindowFunc callback can safely return a window + /// owner from a worker thread without ever touching off-UI. + /// Only needed on .NET Framework where the legacy SetIWin32WindowFunc API is used. + /// +#if NETFRAMEWORK + private sealed class Win32WindowHandle : IWin32Window + { + private readonly IntPtr _hwnd; + public Win32WindowHandle(IntPtr hwnd) => _hwnd = hwnd; + public IntPtr Handle => _hwnd; + } +#endif + + #endregion + + // ────────────────────────────────────────────────────────────────── + #region Private Fields + + /// + /// The form's Win32 window handle, captured on the UI thread in the constructor. + /// Read from worker threads by the Entra ID provider callbacks to parent MSAL's + /// sign-in / WAM broker UI without illegally touching . + /// + private readonly IntPtr _ownerHwnd; + + #endregion + } +} diff --git a/doc/apps/AzureSqlConnector/ModeSelectorForm.cs b/doc/apps/AzureSqlConnector/ModeSelectorForm.cs new file mode 100644 index 0000000000..651412de32 --- /dev/null +++ b/doc/apps/AzureSqlConnector/ModeSelectorForm.cs @@ -0,0 +1,116 @@ +using System; +using System.Drawing; +using System.Windows.Forms; + +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + /// + /// Choice exposed by . + /// + internal enum ConnectionMode + { + /// + /// Use , which calls SqlConnection.OpenAsync() on the UI + /// thread. Relies on the WinForms SynchronizationContext to keep the message pump alive. + /// + UiThreadOpenAsync, + + /// + /// Use , which calls SqlConnection.Open() inside + /// Task.Run on a thread-pool worker. The captured form HWND is passed to MSAL. + /// + WorkerThreadOpen, + } + + /// + /// Tiny modal dialog shown at startup that lets the user pick which connector form + /// (UI-thread async or worker-thread sync) to launch. + /// + internal sealed class ModeSelectorForm : Form + { + private readonly RadioButton _rdoUiThread; + private readonly RadioButton _rdoWorker; + + internal ConnectionMode SelectedMode => + _rdoWorker.Checked ? ConnectionMode.WorkerThreadOpen : ConnectionMode.UiThreadOpenAsync; + + internal ModeSelectorForm() + { + Text = "Azure SQL Connector — Choose Mode"; + FormBorderStyle = FormBorderStyle.FixedDialog; + StartPosition = FormStartPosition.CenterScreen; + MaximizeBox = false; + MinimizeBox = false; + ClientSize = new Size(460, 200); + + Label lblHeader = new Label + { + AutoSize = false, + Text = "Select how SqlConnection.Open should be invoked:", + Location = new Point(16, 14), + Size = new Size(420, 20), + Font = new Font(Font, FontStyle.Bold), + }; + + _rdoUiThread = new RadioButton + { + Text = "&UI thread", + Location = new Point(20, 42), + Size = new Size(420, 20), + Checked = true, + }; + + Label lblUiHint = new Label + { + AutoSize = false, + Text = " Async/Sync open on the UI thread; SynchronizationContext keeps the form responsive.", + Location = new Point(20, 62), + Size = new Size(420, 18), + ForeColor = SystemColors.GrayText, + }; + + _rdoWorker = new RadioButton + { + Text = "&Worker thread", + Location = new Point(20, 90), + Size = new Size(420, 20), + }; + + Label lblWorkerHint = new Label + { + AutoSize = false, + Text = " Sync open on a thread-pool worker; HWND is captured up-front for MSAL.", + Location = new Point(20, 110), + Size = new Size(420, 18), + ForeColor = SystemColors.GrayText, + }; + + Button btnOk = new Button + { + Text = "&Launch", + DialogResult = DialogResult.OK, + Location = new Point(268, 152), + Size = new Size(82, 28), + }; + + Button btnCancel = new Button + { + Text = "Cancel", + DialogResult = DialogResult.Cancel, + Location = new Point(358, 152), + Size = new Size(82, 28), + }; + + AcceptButton = btnOk; + CancelButton = btnCancel; + + Controls.AddRange(new Control[] + { + lblHeader, + _rdoUiThread, lblUiHint, + _rdoWorker, lblWorkerHint, + btnOk, btnCancel, + }); + } + } +} diff --git a/doc/apps/AzureSqlConnector/Program.cs b/doc/apps/AzureSqlConnector/Program.cs new file mode 100644 index 0000000000..c4bfad836c --- /dev/null +++ b/doc/apps/AzureSqlConnector/Program.cs @@ -0,0 +1,39 @@ +using System; +using System.Windows.Forms; + +namespace Microsoft.Data.SqlClient.Samples.AzureSqlConnector +{ + /// + /// Application entry point for the Azure SQL Connector WinForms test app. + /// + internal static class Program + { + /// + /// The main entry point for the application. Shows a small chooser dialog at startup so + /// the user can pick between the UI-thread and the worker-thread + /// variant of the connector. + /// + [STAThread] + private static void Main() + { + Application.EnableVisualStyles(); + Application.SetCompatibleTextRenderingDefault(false); + + ConnectionMode mode; + using (ModeSelectorForm selector = new ModeSelectorForm()) + { + if (selector.ShowDialog() != DialogResult.OK) + { + return; + } + mode = selector.SelectedMode; + } + + Form main = mode == ConnectionMode.WorkerThreadOpen + ? (Form)new MainFormWorker() + : new MainForm(); + + Application.Run(main); + } + } +} diff --git a/doc/apps/AzureSqlConnector/README.md b/doc/apps/AzureSqlConnector/README.md new file mode 100644 index 0000000000..62f100e966 --- /dev/null +++ b/doc/apps/AzureSqlConnector/README.md @@ -0,0 +1,137 @@ +# Azure SQL Connector (WinForms) + +A small Windows Forms test application that lets a user fill in Azure SQL Database connection +parameters in a UI, builds the corresponding ADO.NET connection string via +`SqlConnectionStringBuilder`, and tests connectivity using `Microsoft.Data.SqlClient`. + +It is intended as a quick, repeatable scratch tool for manually validating connection-string +combinations (server / database / authentication mode / encryption / etc.) against an Azure SQL DB +or SQL Server instance, **and as a manual repro** for the WAM-broker behavior added in this +branch's `ActiveDirectoryAuthenticationProvider`. + +The sample multi-targets: + +| TFM | Purpose | +| ---------------- | ------------------------------------------------------------------------------------------------ | +| `net481` | Exercises the legacy `SetIWin32WindowFunc` API used by .NET Framework callers with WinForms. | +| `net10.0-windows` | Exercises the modern `SetParentActivityOrWindowFunc` API used on .NET 8+. | + +`net10.0-windows` restores and builds cleanly on Linux/macOS hosts even though the resulting +binary only runs on Windows, so the project no longer needs a separate no-op cross-platform +fallback. + +> **Note:** `SetParentActivityOrWindowFunc` is also available on `net481` and is the +> recommended API for new code on any framework. The sample wires `net481` up to +> `SetIWin32WindowFunc` only to keep coverage of that legacy code path; replacing the +> `SetIWin32WindowFunc(() => this)` call with `SetParentActivityOrWindowFunc(() => this.Handle)` +> on `net481` works the same way. + +## Mode selector + +When the app launches it shows a small `ModeSelectorForm` that picks between two top-level forms: + +| Mode | Form | What it exercises | +| ---------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------- | +| **UI thread (`OpenAsync`)** | `MainForm` | Calls `SqlConnection.OpenAsync()` on the UI thread so the Windows Forms message pump stays alive during MSAL sign-in. | +| **Worker thread (`Open`, sync)** | `MainFormWorker` | Calls `SqlConnection.Open()` on a background worker thread; the parent window handle is captured up-front on the UI thread. | + +Both forms demonstrate the supported patterns for parenting the WAM broker (or the legacy +embedded WebView on .NET Framework). + +## Form inputs + +| Field | Maps to connection string keyword | +| -------------------------- | ----------------------------------------------- | +| Server name | `Data Source` | +| Database name | `Initial Catalog` *(only added when non-empty)* | +| Authentication | `Authentication` *(SqlAuthenticationMethod)* | +| User ID | `User ID` | +| Password | `Password` | +| Encrypt | `Encrypt` *(Mandatory / Optional / Strict)* | +| Trust server certificate | `TrustServerCertificate` | +| Connect timeout (s) | `Connect Timeout` | + +The **Authentication** dropdown is populated from every member of +`Microsoft.Data.SqlClient.SqlAuthenticationMethod`. The User ID and Password fields are enabled / +disabled automatically based on the selected method: + +- **SqlPassword** / **ActiveDirectoryPassword** — both User ID and Password are required. +- **ActiveDirectoryServicePrincipal** — User ID = App (Client) ID, Password = client secret. +- **ActiveDirectoryManagedIdentity / MSI / Default / Interactive / DeviceCodeFlow / WorkloadIdentity** + — User ID is optional (e.g. user-assigned MI client id), Password is disabled. +- **ActiveDirectoryIntegrated** — credentials come from the OS, both fields disabled. + +## Buttons + +| Button | Action | +| ----------------------- | ---------------------------------------------------------------------- | +| Build Connection String | Builds the connection string from the form values and displays it. | +| Test Connection | Builds the connection string and opens the connection. | +| Copy to Clipboard | Copies the currently-built connection string to the clipboard. | +| Clear All | Resets every input field to its default state. | +| Who Am I? | Connects and runs an identity query (`SUSER_SNAME()`, `ORIGINAL_LOGIN()`, `USER_NAME()`, `DB_NAME()`, `@@SPID`, etc.) and prints the results. | + +The result pane shows the built connection string with the password masked, the test connection +outcome (including SQL error number when applicable), and the server version on success. + +## Prerequisites + +- Visual Studio 2026 (or any IDE / SDK with .NET Framework **4.8.1** Developer Pack installed) for + the `net481` target. The `net10.0-windows` target only needs the .NET 10 SDK. +- Network connectivity to your Azure SQL Database (server firewall must allow your client IP). +- For Entra ID authentication modes, valid credentials available through Azure CLI / environment + variables / managed identity / the WAM broker, depending on the chosen method. + +## Build & run + +From the project folder: + +```pwsh +dotnet build .\AzureSqlConnector.csproj +dotnet run --project .\AzureSqlConnector.csproj -f net10.0-windows # modern WAM API +dotnet run --project .\AzureSqlConnector.csproj -f net481 # legacy IWin32Window API +``` + +Or load `src\Microsoft.Data.SqlClient.slnx` in Visual Studio, set **AzureSqlConnector** as the +startup project, and press **F5**. + +## Example + +1. **Server name:** `myserver.database.windows.net` +2. **Database name:** `MyDb` +3. **Authentication:** `SqlPassword` +4. **User ID:** `sqladmin` +5. **Password:** *your password* +6. **Encrypt:** `Mandatory` +7. **Trust server certificate:** unchecked +8. Click **Test Connection** — the result pane should display + `Connected successfully! Server version: 12.00.xxxx`. + +## Entra ID parent-window plumbing + +For any `ActiveDirectory*` authentication method (especially **ActiveDirectoryInteractive**) the +app installs an `ActiveDirectoryAuthenticationProvider` and tells it which window should host the +sign-in UI: + +- On **`net481`** the form calls `provider.SetIWin32WindowFunc(() => this)`. This is the legacy + API used by .NET Framework callers with the embedded WebView. +- On **`net10.0-windows`** the form calls + `provider.SetParentActivityOrWindowFunc(() => this.Handle)`. This is the modern API that also + integrates with the WAM broker on Windows. + +The provider is registered for every `SqlAuthenticationMethod.ActiveDirectory*` value at startup. + +### Threading patterns + +| Form | Open mode | Parent window callback | +| ----------------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `MainForm` | `OpenAsync` on UI thread | Callback runs on the UI thread when MSAL invokes it, so `this`/`this.Handle` is naturally safe to access. | +| `MainFormWorker` | `Open` (sync) on worker | The form captures `this.Handle` into a field on the UI thread before kicking off the worker; the callback closes over that captured value so it never needs to marshal back. | + +Without one of these patterns the WAM broker (or the embedded WebView on .NET Framework) can fail +to render or stay unresponsive while it waits for the user. + +## Notes + +- This is a sample / diagnostic tool, **not** a product. It does not persist credentials. +- From the repo root: `dotnet run --project .\doc\apps\AzureSqlConnector\AzureSqlConnector.csproj` diff --git a/doc/design-notes/4001-delegated-transaction-reset.md b/doc/design-notes/4001-delegated-transaction-reset.md new file mode 100644 index 0000000000..79d24fb04c --- /dev/null +++ b/doc/design-notes/4001-delegated-transaction-reset.md @@ -0,0 +1,322 @@ +# Root cause analysis: #4001 — pooled connection broken after `TransactionScope` rollback + +| | | +|---|---| +| **Issue** | [#4001](https://github.com/dotnet/SqlClient/issues/4001) | +| **Regressed by** | [#3019](https://github.com/dotnet/SqlClient/pull/3019) (`0322d44c7`), shipped in 6.1.0 | +| **Affected** | 6.1.0 → 6.1.6, `main` | +| **Last good** | 6.0.5 | +| **Code** | `SqlConnectionInternal.ResetConnection()` | + +This note records why the bug happens, why the previous fix caused it, why the new +condition cannot reintroduce the issue that fix was addressing, and — importantly — +what the existing test suite does and does not actually verify. + +--- + +## 1. Background: two ways a connection can be "in" a transaction + +This distinction is the crux of the entire bug. + +When a connection participates in a `TransactionScope`, it ends up in one of **two +distinct** states. They overlap, but neither implies the other. + +### Delegated root — "I *own* this transaction" + +Only one connection is involved, so `System.Transactions` delegates the transaction +down to SQL Server rather than paying for a distributed coordinator. The transaction +lives **on** the connection. + +- `IsTransactionRoot` → `true` +- `EnlistedTransaction` → set at first, then **cleared** later (see below) + +`IsTransactionRoot` is not a stored flag. It is derived: + +```csharp +internal bool IsTransactionRoot => DelegatedTransaction?.IsActive == true; +``` + +Enlistment sets `EnlistedTransaction` unconditionally, so a *freshly* delegated root +has both. But once the transaction is no longer `Active`, +`DbConnectionInternal.DetachCurrentTransactionIfEnded` clears `EnlistedTransaction`: + +```csharp +transactionIsDead = enlistedTransaction.TransactionInformation.Status != TransactionStatus.Active; +if (transactionIsDead) { DetachTransaction(enlistedTransaction, true); } +``` + +The delegated transaction, meanwhile, can still report `IsActive == true`. That +transient **half-state** — root, but no `EnlistedTransaction` — is precisely the state +issue #4001 reproduces in. + +### Enlisted participant — "I *joined* someone else's transaction" + +Multiple resources are involved, so a coordinator (MSDTC) owns the transaction and +each connection enlists in it. + +- `IsTransactionRoot` → `false` +- `EnlistedTransaction` → set + +### The trap + +Neither field subsumes the other: a delegated root can have a `null` +`EnlistedTransaction`, and an enlisted participant is never a root. Any check that +tests only one of these fields silently misses the other case — and does so with no +exception at the point of the mistake. + +--- + +## 2. Where the damage occurs + +When a connection is closed it returns to the pool and is **reset** — wiped clean for +the next consumer. If a transaction is still in flight, that reset must *preserve* it. +The entire decision is one boolean: + +```csharp +_parser.PrepareResetConnection(preserveTransaction); +``` + +Pass `false` while a transaction is genuinely live, and the TDS reset destroys the +server-side transaction **while `System.Transactions` still believes it exists**. + +The failure then surfaces later, some distance from the cause: + +1. `TransactionScope` disposes and rolls back. +2. `SqlDelegatedTransaction.Rollback` asks the server to roll back a transaction the + server no longer has. +3. The rollback fails, and SqlClient calls `DoomThisConnection()`. +4. The physical connection is now permanently marked broken. +5. With a small pool (the report used `MaxPoolSize=1`) that same doomed connection is + immediately handed back out. +6. The next caller gets: + +> `InvalidOperationException: The requested operation cannot be completed because the connection has been broken.` + +The exception names the connection, not the reset that ruined it. That distance +between cause and symptom is what makes this class of bug hard to trace. + +--- + +## 3. What PR #3019 actually changed + +The relevant diff from `0322d44c7`: + +```diff +- _parser.PrepareResetConnection(IsTransactionRoot && !IsNonPoolableTransactionRoot); ++ _parser.PrepareResetConnection(EnlistedTransaction is not null && Pool is not null); +``` + +The old helper was: + +```csharp +internal protected override bool IsNonPoolableTransactionRoot + => IsTransactionRoot && (!Is2008OrNewer || Pool == null); +``` + +Substituting, the pre-#3019 condition was: + +```csharp +IsTransactionRoot && Is2008OrNewer && Pool != null +``` + +The `Is2008OrNewer` term is **deliberately not carried forward**, for two reasons. + +First, it is outside the support matrix. The term is `false` for exactly one server +version — SQL Server 2005 — and the driver's supported floor is SQL Server 2012. + +Second, and more importantly, **it was never safe on its own.** `IsNonPoolableTransactionRoot` +had two jobs, not one. Besides suppressing the preserve bit, it also drove pool routing: +`DbConnectionPool` sent any connection it flagged into *stasis* rather than back into the +pool. Being parked is what made a plain reset harmless — the connection was never handed +to another caller. #3019 deleted the property entirely, and today's pools route on +`EnlistedTransaction` alone. A delegated root with a `null` `EnlistedTransaction` now goes +straight back to the **general** pool. + +Reinstating only the suppression half would therefore reset a live delegated transaction +*and* hand the connection out again — #4001 exactly, just narrowed to SQL Server 2005. +Half of a retired safety mechanism is worse than none of it. + +So the two conditions were: + +| | Question it asked | Covered | Missed | +|---|---|---|---| +| **Pre-#3019** | "Am I the *owner*?" | delegated root | enlisted → **#2970** | +| **#3019** | "Am I *enlisted*?" | enlisted | delegated root → **#4001** | + +**#3019 swapped one case for the other rather than covering both.** It genuinely fixed +#2970, and it traded it for #4001. Both conditions were half-right; neither was wrong +about the case it did cover. + +This reframes the fix: the goal is not to undo #3019, it is to finish it. + +--- + +## 4. The fix + +The predicate is extracted into a helper so it can be tested directly: + +```csharp +internal static bool ShouldPreserveTransactionOnReset( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction) +{ + if (!isPooled) + { + return false; + } + + return isTransactionRoot || hasEnlistedTransaction; +} +``` + +This is `OLD || NEW`: each arm answers one of the two questions from section 1, so the +predicate is `true` wherever either predecessor was. + +--- + +## 5. Why this cannot reintroduce #2970 + +This is the question that matters most, and it is answerable by inspection rather than +by testing. Every reachable state, for a pooled connection: + +| `IsTransactionRoot` | `EnlistedTransaction` | Pre-#3019 | #3019 | **This fix** | +|:---:|:---:|:---:|:---:|:---:| +| `false` | `null` | `false` | `false` | `false` | +| **`true`** | **`null`** | ✅ `true` | ❌ `false` ← **#4001** | ✅ **`true`** | +| **`false`** | **set** | ❌ `false` ← **#2970** | ✅ `true` | ✅ **`true`** | +| `true` | set | `true` | `true` | `true` | + +Read the **#2970 row**. That is the row PR #3019 was created to fix, and this fix still +evaluates `true` there. It is untouched. + +Reintroducing #2970 would require that cell to flip to `false`, and `A || B` cannot +evaluate `false` while `B` is `true`. The guarantee is structural, not empirical. + +Because each arm reproduces its original predecessor exactly, the condition returns +`true` wherever either predecessor did, and never returns `false` where one of them +returned `true`. + +--- + +## 6. How the root cause was established + +The cause was **proven at runtime, not inferred**. The driver was temporarily +instrumented at the reset site and at `DoomThisConnection()`. The captured state at the +critical reset: + +``` +[RESET] obj=4 preserve=False root=False deleg=null enlisted=null pool=set +[RESET] obj=7 preserve=False root=False deleg=null enlisted=null pool=set +[RESET] obj=7 preserve=False root=True deleg=active=True enlisted=null pool=set <-- old: true, new: false +[DOOM] obj=7 + at Microsoft.Data.SqlClient.SqlDelegatedTransaction.Rollback(...) + at System.Transactions.Transaction.Rollback() + at System.Transactions.TransactionScope.InternalDispose() + at System.Transactions.TransactionScope.Dispose() +[FAIL] Bug reproduced +``` + +The third reset is the bug caught in the act: `root=True`, `deleg.IsActive=True`, +`enlisted=null`, and `preserve=False`. A live delegated transaction being discarded. + +This mattered, because **the initial hypothesis was wrong.** The first theory was that +#3019 had made the condition *too broad*, and a narrowing fix was written on that +basis. It did not work. The instrumentation showed the opposite — #3019 had *narrowed* +the condition, not widened it — and the fix was rewritten accordingly. Without runtime +evidence this would have been fixed in the wrong direction. + +All instrumentation was removed before commit. + +--- + +## 7. What the existing tests actually verify + +This section is deliberately blunt, because the intuitive answer is wrong. + +### Bisection + +Against the reporter's reproduction (NHibernate 5.5.2, `MaxPoolSize=1`, +`TransactionScope` with a failed DTC promotion): + +**6.0.5 ✅ · 6.1.0 ❌ · 6.1.1 ❌ · 6.1.4 ❌ · `main` ❌** + +This places the regression in the 6.1.0 window, consistent with #3019. + +### Both pool implementations, both directions + +| | without fix | with fix | +|---|---|---| +| `WaitHandleDbConnectionPool` (default) | ❌ reproduces | ✅ passes | +| `ChannelDbConnectionPool` (`UseConnectionPoolV2`) | ❌ reproduces | ✅ passes | + +The **left-hand column is the load-bearing one.** It was produced by stashing the fix +and rebuilding. Without it, a pass on the V2 pool could simply mean V2 never reaches +this code path, which would prove nothing. + +(V2 in released 6.1.4 throws `NotImplementedException`, so only `main` was testable.) + +### Mutation testing of the manual suite + +`--filter "FullyQualifiedName~TransactionTest"` reports **9/9 passing** with the fix. +That number is easy to over-read, so the suite was mutation-tested: the condition was +replaced with each known-buggy variant and the suite re-run. + +| Condition compiled in | Bug it contains | Suite result | +|---|---|---| +| Pre-#3019 (`IsTransactionRoot && Pool is not null`) | **#2970** | **9/9 passed** | +| #3019 (`EnlistedTransaction is not null && Pool is not null`) | **#4001** | **9/9 passed** | +| This fix (union) | none | 9/9 passed | + +**The suite passes on all three.** It does not detect either bug, and therefore does +not guard this line at all in this environment. + +`Test_EnlistedTransactionPreservedWhilePooled` — the test added by #3019 specifically +to cover #2970 — is tagged `[Trait("Category", "flaky")]` and passes against code that +carries the #2970 bug. + +The practical conclusion: **the 9/9 result is evidence of no collateral damage, not +evidence that the fix works.** + +### The regression test that was added + +An end-to-end reproduction was attempted extensively and abandoned. The `#4001` state +requires a narrow simultaneity — the transaction's status already non-`Active` (so +`DetachCurrentTransactionIfEnded` has cleared `EnlistedTransaction`) while +`DelegatedTransaction.IsActive` is still `true` — and which of two teardown paths in +`SqlDelegatedTransaction` wins is a race: + +- `TransactionEnded` sets `_active = false` and *immediately* calls + `DoomThisConnection()`. If this path runs first, the delegate is already inactive + before any reset, so the state is never observed. +- `Rollback`, driven from `TransactionScope.Dispose`, sets `_active = false` *after* the + reset. This is the ordering the reporter hit. + +Roughly twenty harness variants — varying pool size, pool implementation, promotion +success, explicit rollback, and parking the delegate in the transacted pool ahead of the +enlistment — consistently drove the first path. A test built on that race would be +flaky, which is the same defect `Test_EnlistedTransactionPreservedWhilePooled` already +demonstrates. + +Instead the predicate was extracted into +`SqlConnectionInternal.ShouldPreserveTransactionOnReset` and pinned directly by +`SqlConnectionInternalResetTransactionTests`, following the existing precedent of +`ResolveLoginTimeout` / `SqlConnectionInternalTimeoutTests`. The tests were themselves +mutation-tested: + +| Condition compiled into the helper | Bug it contains | New tests | +|---|---|---| +| `hasEnlistedTransaction` (#3019) | **#4001** | ❌ 1 failed | +| `isTransactionRoot` (pre-#3019) | **#2970** | ❌ 1 failed | +| The shipped condition | none | ✅ 8 passed | + +This satisfies "fails before the change, passes after" for **both** regressions, and +does so deterministically and without a server. + +--- + +## 8. Related + +- **#2970** — the issue #3019 was fixing. Fully preserved by this change (section 5). +- **#2285** — reports the same exception with no reproduction. Plausibly the same root + cause, though unconfirmed. diff --git a/doc/samples/AzureKeyVaultProviderLegacyExample_2_0.cs b/doc/samples/AzureKeyVaultProviderLegacyExample_2_0.cs deleted file mode 100644 index d397de6441..0000000000 --- a/doc/samples/AzureKeyVaultProviderLegacyExample_2_0.cs +++ /dev/null @@ -1,380 +0,0 @@ -/** - * TODO: This sample file should be deleted as the AKV Provider Ctor API is no longer supported with supported versions of AKV provider and MDS. - * Depends on: Delete documentation and sample reference in MS Docs first: https://learn.microsoft.com/en-us/sql/connect/ado-net/sql/azure-key-vault-example?view=sql-server-ver17#legacy-callback-implementation-design-example-with-v20 - * - -//< Snippet1> -using System; -using System.Collections.Generic; -using System.IdentityModel.Tokens.Jwt; -using System.Linq; -using System.Net.Http; -using System.Security.Cryptography; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core; -using Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.Data.SqlClient.Samples -{ - public class AzureKeyVaultProviderLegacyExample_2_0 - { - const string s_algorithm = "RSA_OAEP"; - - // ********* Provide details here *********** - static readonly string s_akvUrl = "https://{KeyVaultName}.vault.azure.net/keys/{Key}/{KeyIdentifier}"; - static readonly string s_clientId = "{Application_Client_ID}"; - static readonly string s_clientSecret = "{Application_Client_Secret}"; - static readonly string s_connectionString = "Server={Server}; Database={database}; Integrated Security=true; Column Encryption Setting=Enabled;"; - // ****************************************** - - public static void Main() - { - // Initialize AKV provider - SqlColumnEncryptionAzureKeyVaultProvider akvProvider = new SqlColumnEncryptionAzureKeyVaultProvider(new LegacyAuthCallbackTokenCredential()); - - // Register AKV provider - SqlConnection.RegisterColumnEncryptionKeyStoreProviders(customProviders: new Dictionary(capacity: 1, comparer: StringComparer.OrdinalIgnoreCase) - { - { SqlColumnEncryptionAzureKeyVaultProvider.ProviderName, akvProvider} - }); - Console.WriteLine("AKV provider Registered"); - - // Create connection to database - using (SqlConnection sqlConnection = new SqlConnection(s_connectionString)) - { - string cmkName = "CMK_WITH_AKV"; - string cekName = "CEK_WITH_AKV"; - string tblName = "AKV_TEST_TABLE"; - - CustomerRecord customer = new CustomerRecord(1, @"Microsoft", @"Corporation"); - - try - { - sqlConnection.Open(); - - // Drop Objects if exists - dropObjects(sqlConnection, cmkName, cekName, tblName); - - // Create Column Master Key with AKV Url - createCMK(sqlConnection, cmkName); - Console.WriteLine("Column Master Key created."); - - // Create Column Encryption Key - createCEK(sqlConnection, cmkName, cekName, akvProvider); - Console.WriteLine("Column Encryption Key created."); - - // Create Table with Encrypted Columns - createTbl(sqlConnection, cekName, tblName); - Console.WriteLine("Table created with Encrypted columns."); - - // Insert Customer Record in table - insertData(sqlConnection, tblName, customer); - Console.WriteLine("Encryted data inserted."); - - // Read data from table - verifyData(sqlConnection, tblName, customer); - Console.WriteLine("Data validated successfully."); - } - finally - { - // Drop table and keys - dropObjects(sqlConnection, cmkName, cekName, tblName); - Console.WriteLine("Dropped Table, CEK and CMK"); - } - - Console.WriteLine("Completed AKV provider Sample."); - } - } - - private static void createCMK(SqlConnection sqlConnection, string cmkName) - { - string KeyStoreProviderName = SqlColumnEncryptionAzureKeyVaultProvider.ProviderName; - - string sql = - $@"CREATE COLUMN MASTER KEY [{cmkName}] - WITH ( - KEY_STORE_PROVIDER_NAME = N'{KeyStoreProviderName}', - KEY_PATH = N'{s_akvUrl}' - );"; - - using (SqlCommand command = sqlConnection.CreateCommand()) - { - command.CommandText = sql; - command.ExecuteNonQuery(); - } - } - - private static void createCEK(SqlConnection sqlConnection, string cmkName, string cekName, SqlColumnEncryptionAzureKeyVaultProvider sqlColumnEncryptionAzureKeyVaultProvider) - { - string sql = - $@"CREATE COLUMN ENCRYPTION KEY [{cekName}] - WITH VALUES ( - COLUMN_MASTER_KEY = [{cmkName}], - ALGORITHM = '{s_algorithm}', - ENCRYPTED_VALUE = {GetEncryptedValue(sqlColumnEncryptionAzureKeyVaultProvider)} - )"; - - using (SqlCommand command = sqlConnection.CreateCommand()) - { - command.CommandText = sql; - command.ExecuteNonQuery(); - } - } - - private static string GetEncryptedValue(SqlColumnEncryptionAzureKeyVaultProvider sqlColumnEncryptionAzureKeyVaultProvider) - { - byte[] plainTextColumnEncryptionKey = new byte[32]; - RandomNumberGenerator rng = RandomNumberGenerator.Create(); - rng.GetBytes(plainTextColumnEncryptionKey); - - byte[] encryptedColumnEncryptionKey = sqlColumnEncryptionAzureKeyVaultProvider.EncryptColumnEncryptionKey(s_akvUrl, s_algorithm, plainTextColumnEncryptionKey); - string EncryptedValue = string.Concat("0x", BitConverter.ToString(encryptedColumnEncryptionKey).Replace("-", string.Empty)); - return EncryptedValue; - } - - private static void createTbl(SqlConnection sqlConnection, string cekName, string tblName) - { - string ColumnEncryptionAlgorithmName = @"AEAD_AES_256_CBC_HMAC_SHA_256"; - - string sql = - $@"CREATE TABLE [dbo].[{tblName}] - ( - [CustomerId] [int] ENCRYPTED WITH (COLUMN_ENCRYPTION_KEY = [{cekName}], ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = '{ColumnEncryptionAlgorithmName}'), - [FirstName] [nvarchar](50) COLLATE Latin1_General_BIN2 ENCRYPTED WITH (COLUMN_ENCRYPTION_KEY = [{cekName}], ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = '{ColumnEncryptionAlgorithmName}'), - [LastName] [nvarchar](50) COLLATE Latin1_General_BIN2 ENCRYPTED WITH (COLUMN_ENCRYPTION_KEY = [{cekName}], ENCRYPTION_TYPE = DETERMINISTIC, ALGORITHM = '{ColumnEncryptionAlgorithmName}') - )"; - - using (SqlCommand command = sqlConnection.CreateCommand()) - { - command.CommandText = sql; - command.ExecuteNonQuery(); - } - } - - private static void insertData(SqlConnection sqlConnection, string tblName, CustomerRecord customer) - { - string insertSql = $"INSERT INTO [{tblName}] (CustomerId, FirstName, LastName) VALUES (@CustomerId, @FirstName, @LastName);"; - - using (SqlTransaction sqlTransaction = sqlConnection.BeginTransaction()) - using (SqlCommand sqlCommand = new SqlCommand(insertSql, - connection: sqlConnection, transaction: sqlTransaction, - columnEncryptionSetting: SqlCommandColumnEncryptionSetting.Enabled)) - { - sqlCommand.Parameters.AddWithValue(@"CustomerId", customer.Id); - sqlCommand.Parameters.AddWithValue(@"FirstName", customer.FirstName); - sqlCommand.Parameters.AddWithValue(@"LastName", customer.LastName); - - sqlCommand.ExecuteNonQuery(); - sqlTransaction.Commit(); - } - } - - private static void verifyData(SqlConnection sqlConnection, string tblName, CustomerRecord customer) - { - // Test INPUT parameter on an encrypted parameter - using (SqlCommand sqlCommand = new SqlCommand($"SELECT CustomerId, FirstName, LastName FROM [{tblName}] WHERE FirstName = @firstName", - sqlConnection)) - { - SqlParameter customerFirstParam = sqlCommand.Parameters.AddWithValue(@"firstName", @"Microsoft"); - customerFirstParam.Direction = System.Data.ParameterDirection.Input; - customerFirstParam.ForceColumnEncryption = true; - - using (SqlDataReader sqlDataReader = sqlCommand.ExecuteReader()) - { - ValidateResultSet(sqlDataReader); - } - } - } - - private static void ValidateResultSet(SqlDataReader sqlDataReader) - { - Console.WriteLine(" * Row available: " + sqlDataReader.HasRows); - - while (sqlDataReader.Read()) - { - if (sqlDataReader.GetInt32(0) == 1) - { - Console.WriteLine(" * Employee Id received as sent: " + sqlDataReader.GetInt32(0)); - } - else - { - Console.WriteLine("Employee Id didn't match"); - } - - if (sqlDataReader.GetString(1) == @"Microsoft") - { - Console.WriteLine(" * Employee Firstname received as sent: " + sqlDataReader.GetString(1)); - } - else - { - Console.WriteLine("Employee FirstName didn't match."); - } - - if (sqlDataReader.GetString(2) == @"Corporation") - { - Console.WriteLine(" * Employee LastName received as sent: " + sqlDataReader.GetString(2)); - } - else - { - Console.WriteLine("Employee LastName didn't match."); - } - } - } - - private static void dropObjects(SqlConnection sqlConnection, string cmkName, string cekName, string tblName) - { - using (SqlCommand cmd = sqlConnection.CreateCommand()) - { - cmd.CommandText = $@"IF EXISTS (select * from sys.objects where name = '{tblName}') BEGIN DROP TABLE [{tblName}] END"; - cmd.ExecuteNonQuery(); - cmd.CommandText = $@"IF EXISTS (select * from sys.column_encryption_keys where name = '{cekName}') BEGIN DROP COLUMN ENCRYPTION KEY [{cekName}] END"; - cmd.ExecuteNonQuery(); - cmd.CommandText = $@"IF EXISTS (select * from sys.column_master_keys where name = '{cmkName}') BEGIN DROP COLUMN MASTER KEY [{cmkName}] END"; - cmd.ExecuteNonQuery(); - } - } - - private class CustomerRecord - { - internal int Id { get; set; } - internal string FirstName { get; set; } - internal string LastName { get; set; } - - public CustomerRecord(int id, string fName, string lName) - { - Id = id; - FirstName = fName; - LastName = lName; - } - } - - private class LegacyAuthCallbackTokenCredential : TokenCredential - { - string _authority = ""; - string _resource = ""; - string _akvUrl = ""; - - public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) => - AcquireTokenAsync().GetAwaiter().GetResult(); - - public override async ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) => - await AcquireTokenAsync(); - - private async Task AcquireTokenAsync() - { - // Added to reduce HttpClient calls. - // For multi-user support, a better design can be implemented as needed. - if (_akvUrl != s_akvUrl) - { - using (HttpClient httpClient = new HttpClient()) - { - HttpResponseMessage response = await httpClient.GetAsync(s_akvUrl); - string challenge = response?.Headers.WwwAuthenticate.FirstOrDefault()?.ToString(); - string trimmedChallenge = ValidateChallenge(challenge); - string[] pairs = trimmedChallenge.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries); - - if (pairs != null && pairs.Length > 0) - { - for (int i = 0; i < pairs.Length; i++) - { - string[] pair = pairs[i]?.Split('='); - - if (pair.Length == 2) - { - string key = pair[0]?.Trim().Trim(new char[] { '\"' }); - string value = pair[1]?.Trim().Trim(new char[] { '\"' }); - - if (!string.IsNullOrEmpty(key)) - { - if (key.Equals("authorization", StringComparison.InvariantCultureIgnoreCase)) - { - _authority = value; - } - else if (key.Equals("resource", StringComparison.InvariantCultureIgnoreCase)) - { - _resource = value; - } - } - } - } - } - } - _akvUrl = s_akvUrl; - } - - string strAccessToken = await AzureActiveDirectoryAuthenticationCallback(_authority, _resource); - DateTime expiryTime = InterceptAccessTokenForExpiry(strAccessToken); - return new AccessToken(strAccessToken, new DateTimeOffset(expiryTime)); - } - - private DateTime InterceptAccessTokenForExpiry(string accessToken) - { - if (null == accessToken) - { - throw new ArgumentNullException(accessToken); - } - - var jwtHandler = new JwtSecurityTokenHandler(); - var jwtOutput = string.Empty; - - // Check Token Format - if (!jwtHandler.CanReadToken(accessToken)) - throw new FormatException(accessToken); - - JwtSecurityToken token = jwtHandler.ReadJwtToken(accessToken); - - // Re-serialize the Token Headers to just Key and Values - var jwtHeader = JsonConvert.SerializeObject(token.Header.Select(h => new { h.Key, h.Value })); - jwtOutput = $"{{\r\n\"Header\":\r\n{JToken.Parse(jwtHeader)},"; - - // Re-serialize the Token Claims to just Type and Values - var jwtPayload = JsonConvert.SerializeObject(token.Claims.Select(c => new { c.Type, c.Value })); - jwtOutput += $"\r\n\"Payload\":\r\n{JToken.Parse(jwtPayload)}\r\n}}"; - - // Output the whole thing to pretty JSON object formatted. - string jToken = JToken.Parse(jwtOutput).ToString(Formatting.Indented); - JToken payload = JObject.Parse(jToken).GetValue("Payload"); - - return new DateTime(1970, 1, 1).AddSeconds((long)payload[4]["Value"]); - } - - private static string ValidateChallenge(string challenge) - { - string Bearer = "Bearer "; - if (string.IsNullOrEmpty(challenge)) - throw new ArgumentNullException(nameof(challenge)); - - string trimmedChallenge = challenge.Trim(); - - if (!trimmedChallenge.StartsWith(Bearer)) - throw new ArgumentException("Challenge is not Bearer", nameof(challenge)); - - return trimmedChallenge.Substring(Bearer.Length); - } - - /// - /// Legacy implementation of Authentication Callback, used by Azure Key Vault provider 1.0. - /// This can be leveraged to support multi-user authentication support in the same Azure Key Vault Provider. - /// - /// Authorization URL - /// Resource - /// - public static async Task AzureActiveDirectoryAuthenticationCallback(string authority, string resource) - { - var authContext = new AuthenticationContext(authority); - ClientCredential clientCred = new ClientCredential(s_clientId, s_clientSecret); - AuthenticationResult result = await authContext.AcquireTokenAsync(resource, clientCred); - if (result == null) - { - throw new InvalidOperationException($"Failed to retrieve an access token for {resource}"); - } - return result.AccessToken; - } - } - } -} -// -*/ diff --git a/doc/samples/Microsoft.Data.SqlClient.Samples.csproj b/doc/samples/Microsoft.Data.SqlClient.Samples.csproj index c194fac4d0..1887c9b73e 100644 --- a/doc/samples/Microsoft.Data.SqlClient.Samples.csproj +++ b/doc/samples/Microsoft.Data.SqlClient.Samples.csproj @@ -10,15 +10,19 @@ False + - - - - - + + + + + + + + diff --git a/doc/samples/SqlDataRecord.cs b/doc/samples/SqlDataRecord.cs index f81de5b850..18981dc7db 100644 --- a/doc/samples/SqlDataRecord.cs +++ b/doc/samples/SqlDataRecord.cs @@ -2,10 +2,9 @@ namespace SqlDataRecordCS; using System; +using System.Collections.Generic; using System.Data; -using System.Data.Sql; -using System.Data.SqlTypes; -using Microsoft.SqlServer.Server; +using Microsoft.Data.SqlClient.Server; public sealed partial class SqlDataRecordTester { @@ -13,41 +12,40 @@ private SqlDataRecordTester() { } -[SqlProcedure] -public static void CallTestMethods() -{ - CreateNewRecord(); - CreateNewRecord1(); - -} - // +//using System; +//using System.Collections.Generic; +//using System.Data; //using Microsoft.Data.SqlClient.Server; -[SqlProcedure] -public static void CreateNewRecord() +// Stream rows to SQL Server as a table-valued parameter. +public static IEnumerable GetRecords() { - - // Variables. - SqlDataRecord record; + // Re-use a single SqlDataRecord instance rather than allocating a new one for each row. + // Each row's values are read before SqlCommand advances to the next one. + SqlDataRecord record; - // Create a new record with the column metadata. The constructor is - // able to accept a variable number of parameters. - record = new SqlDataRecord(new SqlMetaData[] { new SqlMetaData("Column1", SqlDbType.NVarChar, 12), + // Create a new record with the column metadata. The constructor is + // able to accept a variable number of parameters. + record = new SqlDataRecord(new SqlMetaData[] { new SqlMetaData("Column1", SqlDbType.NVarChar, 12), new SqlMetaData("Column2", SqlDbType.Int), new SqlMetaData("Column3", SqlDbType.DateTime) }); - // Set the record fields. - record.SetString(0, "Hello World!"); - record.SetInt32(1, 42); - record.SetDateTime(2, DateTime.Now); + // Set the record fields. + record.SetString(0, "Hello World!"); + record.SetInt32(1, 42); + record.SetDateTime(2, DateTime.Now); - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Stream the first record to SQL Server. + yield return record; + + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } // -public static void CreateNewRecord1() +public static void CreateNewRecord() { // @@ -65,15 +63,11 @@ public static void CreateNewRecord1() // Create a new record with the column metadata. record = new SqlDataRecord(new SqlMetaData[] { column1Info, column2Info }); -// - // Set the record fields. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); -// Send the record to the calling program. -SqlContext.Pipe.Send(record); - +// } } #endif diff --git a/doc/samples/SqlMetaData.cs b/doc/samples/SqlMetaData.cs index c23cf363b7..fd6e2fff6b 100644 --- a/doc/samples/SqlMetaData.cs +++ b/doc/samples/SqlMetaData.cs @@ -2,10 +2,9 @@ namespace SqlMetaDataCS; using System; +using System.Collections.Generic; using System.Data; -using System.Data.Sql; -using System.Data.SqlTypes; -using Microsoft.SqlServer.Server; +using Microsoft.Data.SqlClient.Server; public sealed partial class SqlMetaDataTester { @@ -14,10 +13,12 @@ private SqlMetaDataTester() } // + // using System; + // using System.Collections.Generic; + // using System.Data; // using Microsoft.Data.SqlClient.Server; - [SqlProcedure] - public static void CreateNewRecord() + public static IEnumerable ReturnNewRecords() { // Variables. SqlMetaData column1Info; @@ -35,13 +36,15 @@ public static void CreateNewRecord() column2Info, column3Info }); - // Set the record fields. + // Set the fields of the first record and stream it to SQL Server. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); + yield return record; - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } // diff --git a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml index 7b0af6f6c9..c8b69e73c9 100644 --- a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml +++ b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml @@ -1,4 +1,4 @@ - + @@ -6,25 +6,28 @@ - This class is used together with to send result sets to the client from managed code stored-procedures. + This class describes a single row of a table-valued parameter. Construct one from an array of objects which describe the column metadata of the record, populate it with the Set<Type> methods, include it in an IEnumerable<SqlDataRecord> and assign that IEnumerable<SqlDataRecord> to . - When writing common language runtime (CLR) applications, you should re-use existing SqlDataRecord objects instead of creating new ones every time. Creating many new SqlDataRecord objects could severely deplete memory and adversely affect performance. + For best performance when sending multiple rows, re-use a single instance across an IEnumerable<SqlDataRecord> iterator: set new column values and the same instance for each row. will read each record before advancing to the next. - The following example shows how to create several objects, which describe the column metadata of a record, and creating a . The column values of the are set and the is sent to the calling program by using the class. + The following example shows how to create several objects describing the column metadata of a record, then to create a re-usable instance. The column values of the are set and the same can be used to stream multiple rows to SQL Server as a table-valued parameter. + using System; + using System.Collections.Generic; + using System.Data; using Microsoft.Data.SqlClient.Server; - - [Microsoft.Data.SqlClient.Server.SqlProcedure] - public static void CreateNewRecord() + + // Stream rows to SQL Server as a table-valued parameter. + public static IEnumerable<SqlDataRecord> CreateNewRecord() { - - // Variables. + // Re-use a single SqlDataRecord instance rather than allocating a new one for each row. + // Each row's values are read before SqlCommand advances to the next one. SqlDataRecord record; // Create a new record with the column metadata. The constructor is @@ -38,8 +41,12 @@ record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Stream the first record to SQL Server. + yield return record; + + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } @@ -75,9 +82,6 @@ // Set the record fields. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); - - // Send the record to the calling program. - SqlContext.Pipe.Send(record); diff --git a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml index cbabfd8cb0..492e85e46b 100644 --- a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml +++ b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml @@ -1,19 +1,24 @@ - + - Specifies and retrieves metadata information from parameters and columns of objects. This class cannot be inherited. + Specifies the metadata information of a column used by a object. This class cannot be inherited. + + instances are typically built once and reused to construct many objects which are streamed to SQL Server as a table-valued parameter. For more information, see Table-Valued Parameters. + - The following example shows the creation of several objects, which describe the column metadata of a record, and the creation of a . The column values of the are set and the is sent to the calling program using the class. + The following example shows the creation of several objects, which describe the column metadata of a record, and the generation of a stream of records. These records can be streamed to SQL Server as a table-valued parameter by assigning the return value of the method to the property. + using System; + using System.Collections.Generic; + using System.Data; using Microsoft.Data.SqlClient.Server; - - [Microsoft.Data.SqlClient.Server.SqlProcedure] - public static void CreateNewRecord() + + public static IEnumerable<SqlDataRecord> ReturnNewRecords() { // Variables. SqlMetaData column1Info; @@ -32,13 +37,15 @@ column2Info, column3Info }); - // Set the record fields. + // Set the fields of the first record and stream it to SQL Server. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); + yield return record; - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } @@ -456,7 +463,7 @@ The SQL Server type name for . - Initializes a new instance of the class with the specified column name, user-defined type (UDT), and SQLServer type. + Initializes a new instance of the class with the specified column name, user-defined type (UDT), and SQL Server type. diff --git a/doc/snippets/Microsoft.Data.SqlClient/ActiveDirectoryAuthenticationProvider.xml b/doc/snippets/Microsoft.Data.SqlClient/ActiveDirectoryAuthenticationProvider.xml index 36f10a6aab..4dd9f61d5a 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/ActiveDirectoryAuthenticationProvider.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/ActiveDirectoryAuthenticationProvider.xml @@ -1,4 +1,9 @@ - + + @@ -74,7 +79,12 @@ Clears cached user tokens from the token provider. - This will cause interactive authentication prompts to appear again if tokens were previously being obtained from the cache. + + This will cause interactive authentication prompts to appear again if tokens were previously being obtained from the cache. + + + The driver's per-pool federated-authentication token cache is also cleared, so subsequent calls will reacquire fed-auth tokens instead of reusing cached entries. + diff --git a/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml b/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml new file mode 100644 index 0000000000..58dc683dcb --- /dev/null +++ b/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml @@ -0,0 +1,147 @@ + + + + + Specifies the known application identifiers that Microsoft.Data.SqlClient reports for user agent telemetry. + + + + Production applications that meet the bar are welcome to reserve an identifier here. + + + Identifier reservations are as follows: + + + + 0x0001-0x7FFF: Microsoft-defined large-scale applications. + + + 0x8000-0xBFFF: Reserved for small-scale use. + + + 0xC000-0xFFFF: Public and developer use. + + + + An unregistered identifier may still be reported by casting a value to this type. + + + + + + No application identity is reported. This is the default. + + + 0 + + + + + The Microsoft Entity Framework Core SQL Server provider. + + + 1 + + + + + Microsoft Semantic Kernel. + + + 2 + + + + + Microsoft SQL Server Management Studio. + + + 3 + + + + + Microsoft SQL Server Management Objects. + + + 4 + + + + + Microsoft SQL Server Data-Tier Application Framework. + + + 5 + + + + + Microsoft SQL Tools Service. + + + 6 + + + + + Microsoft ASP.NET Core distributed SQL Server cache. + + + 7 + + + + + Microsoft Entity Framework 6 SQL Server provider. + + + 8 + + + + + Microsoft Azure Functions SQL extension. + + + 9 + + + + + Microsoft Orleans ADO.NET providers. + + + 10 + + + + + Microsoft Durable Task SQL Server provider. + + + 11 + + + + + The sqlpackage command-line tool. + + + 12 + + + sqlpackage is built on the Data-Tier Application Framework, but reports its own identifier so that + command-line use can be told apart from other callers of that framework. + + + + + Microsoft Data API builder. + + + 13 + + + + diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBatch.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBatch.xml index 40f34cba48..d2fdff51d2 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBatch.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBatch.xml @@ -1,4 +1,4 @@ - + @@ -179,7 +179,7 @@ The list of contained in the batch in a . - + Sends the to the and builds a . @@ -232,7 +232,85 @@ - + + + An instance of , specifying options for batch execution and data retrieval. + Only and are supported at the batch level. + + + Sends the to the and builds a . + + + + The following example creates a and a , then adds multiple objects to the batch. It then executes the batch, creating a . The example reads through the results of the batch commands, writing them to the console. Finally, the example closes the and then the as the using blocks fall out of scope. + + + using Microsoft.Data.SqlClient; + + class Program + { + static void Main() + { + string str = "Data Source=(local);Initial Catalog=Northwind;" + + "Integrated Security=SSPI;Encrypt=False"; + RunBatch(str); + } + + static void RunBatch(string connString) + { + using var connection = new SqlConnection(connString); + connection.Open(); + + using var batch = new SqlBatch(connection); + + const int count = 10; + const string parameterName = "parameter"; + for (int i = 0; i < count; i++) + { + var batchCommand = new SqlBatchCommand($"SELECT @{parameterName} as value"); + batchCommand.Parameters.Add(new SqlParameter(parameterName, i)); + batch.BatchCommands.Add(batchCommand); + } + + var results = new List<int>(count); + using (SqlDataReader reader = batch.ExecuteReader(CommandBehavior.CloseConnection)) + { + do + { + while (reader.Read()) + { + results.Add(reader.GetFieldValue<int>(0)); + } + } while (reader.NextResult()); + } + Console.WriteLine(string.Join(", ", results)); + } + } + + + + + A token to cancel the asynchronous operation. + + An asynchronous version of , which sends the to the and builds a . + Exceptions will be reported via the returned Task object. + + A task representing the asynchronous operation. + + An error occurred while executing the batch. + + + The value is invalid. + + + The cancellation token was canceled. This exception is stored into the returned task. + + + + + An instance of , specifying options for batch execution and data retrieval. + Only and are supported at the batch level. + A token to cancel the asynchronous operation. An asynchronous version of , which sends the to the and builds a . @@ -303,6 +381,7 @@ An instance of , specifying options for batch execution and data retrieval. + Only and are supported at the batch level. Executes the batch against its connection, returning a which can be used to access the results. @@ -326,7 +405,10 @@ When the batch returns multiple result sets from different commands, - One of the enumeration values that specifies options for batch execution and data retrieval. + + An instance of , specifying options for batch execution and data retrieval. + Only and are supported at the batch level. + A token to cancel the asynchronous operation. This implementation invokes the method and returns a completed task. The default implementation will return a cancelled task if passed an already cancelled cancellation token. This method accepts a cancellation token that can be used to request the operation to be cancelled early. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlBatchCommand.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlBatchCommand.xml index 8f0be84d25..5145571840 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlBatchCommand.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlBatchCommand.xml @@ -1,4 +1,4 @@ - + @@ -87,7 +87,8 @@ - One of the values, indicating options for statement execution and data retrieval. + An instance of , specifying options for statement execution and data retrieval. + Only and are supported at the statement level. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlColumnEncryptionKeyStoreProvider.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlColumnEncryptionKeyStoreProvider.xml index 401258f9e5..c3990851c5 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlColumnEncryptionKeyStoreProvider.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlColumnEncryptionKeyStoreProvider.xml @@ -11,51 +11,159 @@ + + Decrypts the specified encrypted value of a column encryption key. + The encrypted value is expected to be encrypted using the column + master key with the specified key path and using the specified + algorithm. + - The master key path. + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). - The encryption algorithm. + The encryption algorithm name. For example, + RSA_OAEP. The encrypted column encryption key. + + Returns a representing the decrypted + column encryption key. + + + - Decrypts the specified encrypted value of a column encryption key. The encrypted value is expected to be encrypted using the column master key with the specified key path and using the specified algorithm. + Asynchronously decrypts the specified encrypted value of a column + encryption key. The encrypted value is expected to be encrypted using + the column master key with the specified key path and using the + specified algorithm. + + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). + + + The encryption algorithm name. For example, + RSA_OAEP. + + + The encrypted column encryption key. + + + A token to cancel the asynchronous operation. + - Returns . The decrypted column encryption key. + A task that returns a representing + the decrypted column encryption key on completion. - + + + The default implementation first checks the + and returns a canceled task + if cancellation has been requested. Otherwise, it calls the + synchronous + + method and wraps the result in a completed task. If the synchronous + method throws, the exception is caught and a faulted task is + returned rather than throwing synchronously. + + + Derived classes that perform I/O (such as calls to Azure Key Vault) + should override this method with a truly asynchronous + implementation that honors the cancellation token. + + + + + Encrypts a column encryption key using the column master key with + the specified key path and using the specified algorithm. + - The master key path. + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). - The encryption algorithm. + The encryption algorithm name. For example, + RSA_OAEP. - The plaintext column encryption key. + The column encryption key to encrypt. + + Returns a representing the encrypted + column encryption key. + + + - Encrypts a column encryption key using the column master key with the specified key path and using the specified algorithm. + Asynchronously encrypts a column encryption key using the column + master key with the specified key path and using the specified + algorithm. + + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). + + + The encryption algorithm name. For example, + RSA_OAEP. + + + The column encryption key to encrypt. + + + A token to cancel the asynchronous operation. + - Returns . The encrypted column encryption key. + A task that returns a representing + the encrypted column encryption key on completion. - + + + The default implementation first checks the + and returns a canceled task + if cancellation has been requested. Otherwise, it calls the + synchronous + + method and wraps the result in a completed task. If the synchronous + method throws, the exception is caught and a faulted task is + returned rather than throwing synchronously. + + + Derived classes that perform I/O (such as calls to Azure Key Vault) + should override this method with a truly asynchronous + implementation that honors the cancellation token. + + + + + When implemented in a derived class, signs the column master key + metadata with the column master key referenced by the + parameter. + - The column master key path. + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). - to indicate that the column master key supports enclave computations; otherwise, . + to indicate that the column master key + supports enclave computations; otherwise, + . When , the + generated signature covers the enclave-enabled property so that + the metadata can later be verified for enclave use. - - When implemented in a derived class, digitally signs the column master key metadata with the column master key referenced by the parameter. The input values used to generate the signature should be the specified values of the and parameters. - - The signature of the column master key metadata. + Returns the signature of the column master key metadata. The format + is provider-specific. @@ -69,23 +177,142 @@ In all cases. + + + When implemented in a derived class, asynchronously signs the column + master key metadata with the column master key referenced by the + parameter. + + + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). + + + to indicate that the column master key + supports enclave computations; otherwise, + . When , the + generated signature covers the enclave-enabled property so that + the metadata can later be verified for enclave use. + + + A token to cancel the asynchronous operation. + + + A task that, when completed, returns a + representing the signature of the column master key metadata. The + signature format is provider-specific. + + + + The default implementation first checks the + and returns a canceled task + if cancellation has been requested. Otherwise, it calls the + synchronous + + method, which throws a + by default. In + this case, the returned task will be faulted with + rather than + throwing synchronously. + + + Derived classes that perform I/O should override this method with + a truly asynchronous implementation that honors the cancellation + token. + + + Key store providers that wish to use enclaves with + Always Encrypted + should override this method with a truly asynchronous + implementation when the signing operation involves I/O. + + + + + When implemented in a derived class, this method is expected to + verify the specified signature is valid for the column master key + with the specified key path and the specified enclave behavior. The + default implementation throws + . + - The column master key path. + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). - Indicates whether the column master key supports enclave computations. + Indicates whether the column master key supports enclave + computations. This value must match what was passed to + + when the signature was generated. - The signature of the column master key metadata. + The signature to verify, as previously returned by + . + The format is provider-specific. + + When implemented in a derived class, the method is expected to + return if the specified signature is valid, + or if the specified signature is not valid. + The default implementation throws + . + + + - When implemented in a derived class, this method is expected to verify the specified signature is valid for the column master key with the specified key path and the specified enclave behavior. The default implementation throws `NotImplementedException`. + When implemented in a derived class, asynchronously verifies the + specified signature is valid for the column master key with the + specified key path and the specified enclave behavior. The default + implementation returns a faulted task with + . + + The column master key path. The path format is specific to the key + store provider implementation (e.g. a thumbprint for the certificate + store, or a key identifier URL for Azure Key Vault). + + + Indicates whether the column master key supports enclave + computations. This value must match what was passed to + + when the signature was generated. + + + The signature to verify, as previously returned by + . + The format is provider-specific. + + + A token to cancel the asynchronous operation. + - When implemented in a derived class, the method is expected to return true if the specified signature is valid, or false if the specified signature is not valid. The default implementation throws `NotImplementedException`. + A task that, when completed, returns if the + specified signature is valid, or if it is + not valid. - + + + The default implementation first checks the + and returns a canceled task + if cancellation has been requested. Otherwise, it calls the + synchronous + + method, which throws a + by default. In + this case, the returned task will be faulted with + rather than + throwing synchronously. + + + Derived classes that perform I/O should override this method with + a truly asynchronous implementation that honors the cancellation + token. + + + Gets or sets the lifespan of the decrypted column encryption key in the cache. Once the timespan has elapsed, the decrypted column encryption key is discarded and must be revalidated. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlCommand.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlCommand.xml index a3d87c843d..34c738b41e 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlCommand.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlCommand.xml @@ -357,35 +357,35 @@ Next, compile and execute the following: using System; using System.Data; using Microsoft.Data.SqlClient; - + class Class1 { static void Main() { - // This is a simple example that demonstrates the usage of the + // This is a simple example that demonstrates the usage of the // BeginExecuteNonQuery functionality. - // The WAITFOR statement simply adds enough time to prove the + // The WAITFOR statement simply adds enough time to prove the // asynchronous nature of the command. - + string commandText = "UPDATE Production.Product SET ReorderPoint = ReorderPoint + 1 " + "WHERE ReorderPoint Is Not Null;" + "WAITFOR DELAY '0:0:3';" + "UPDATE Production.Product SET ReorderPoint = ReorderPoint - 1 " + "WHERE ReorderPoint Is Not Null"; - + RunCommandAsynchronously(commandText, GetConnectionString()); - + Console.WriteLine("Press ENTER to continue."); Console.ReadLine(); } - + private static void RunCommandAsynchronously(string commandText, string connectionString) { // Given command text and connection string, asynchronously execute // the specified command against the connection. For this example, - // the code displays an indicator as it is working, verifying the - // asynchronous behavior. + // the code displays an indicator as it is working, verifying the + // asynchronous behavior. using (SqlConnection connection = new SqlConnection(connectionString)) { try @@ -393,17 +393,17 @@ Next, compile and execute the following: int count = 0; SqlCommand command = new SqlCommand(commandText, connection); connection.Open(); - + IAsyncResult result = command.BeginExecuteNonQuery(); while (!result.IsCompleted) { Console.WriteLine("Waiting ({0})", count++); // Wait for 1/10 second, so the counter - // does not consume all available resources + // does not consume all available resources // on the main thread. System.Threading.Thread.Sleep(100); } - + Console.WriteLine("Command complete. Affected {0} rows.", command.EndExecuteNonQuery(result)); } @@ -423,11 +423,11 @@ Next, compile and execute the following: } } } - + private static string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=SSPI;" + "Initial Catalog=AdventureWorks"; } @@ -527,7 +527,7 @@ Next, compile and execute the following: using System.Text; using System.Windows.Forms; using Microsoft.Data.SqlClient; - + namespace Microsoft.AdoDotNet.CodeSamples { public partial class Form1 : Form @@ -536,9 +536,9 @@ Next, compile and execute the following: { InitializeComponent(); } - - // Hook up the form's Load event handler (you can double-click on - // the form's design surface in Visual Studio), and then add + + // Hook up the form's Load event handler (you can double-click on + // the form's design surface in Visual Studio), and then add // this code to the form's class: private void Form1_Load(object sender, EventArgs e) { @@ -546,42 +546,42 @@ Next, compile and execute the following: this.FormClosing += new System.Windows.Forms. FormClosingEventHandler(this.Form1_FormClosing); } - + // You need this delegate in order to display text from a thread // other than the form's thread. See the HandleCallback // procedure for more information. - // This same delegate matches both the DisplayStatus + // This same delegate matches both the DisplayStatus // and DisplayResults methods. private delegate void DisplayInfoDelegate(string Text); - + // This flag ensures that the user does not attempt - // to restart the command or close the form while the + // to restart the command or close the form while the // asynchronous command is executing. private bool isExecuting; - - // This example maintains the connection object + + // This example maintains the connection object // externally, so that it is available for closing. private SqlConnection connection; - + private static string GetConnectionString() { - // To avoid storing the connection string in your code, + // To avoid storing the connection string in your code, // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; } - + private void DisplayStatus(string Text) { this.label1.Text = Text; } - + private void DisplayResults(string Text) { this.label1.Text = Text; DisplayStatus("Ready"); } - + private void Form1_FormClosing(object sender, System.Windows.Forms.FormClosingEventArgs e) { if (isExecuting) @@ -591,7 +591,7 @@ Next, compile and execute the following: e.Cancel = true; } } - + private void button1_Click(object sender, System.EventArgs e) { if (isExecuting) @@ -608,7 +608,7 @@ Next, compile and execute the following: DisplayResults(""); DisplayStatus("Connecting..."); connection = new SqlConnection(GetConnectionString()); - // To emulate a long-running query, wait for + // To emulate a long-running query, wait for // a few seconds before working with the data. // This command does not do much, but that's the point-- // it does not change your data, in the long run. @@ -618,19 +618,19 @@ Next, compile and execute the following: "WHERE ReorderPoint Is Not Null;" + "UPDATE Production.Product SET ReorderPoint = ReorderPoint - 1 " + "WHERE ReorderPoint Is Not Null"; - + command = new SqlCommand(commandText, connection); connection.Open(); - + DisplayStatus("Executing..."); isExecuting = true; - // Although it is not required that you pass the - // SqlCommand object as the second parameter in the + // Although it is not required that you pass the + // SqlCommand object as the second parameter in the // BeginExecuteNonQuery call, doing so makes it easier // to call EndExecuteNonQuery in the callback procedure. AsyncCallback callback = new AsyncCallback(HandleCallback); command.BeginExecuteNonQuery(callback, command); - + } catch (Exception ex) { @@ -643,7 +643,7 @@ Next, compile and execute the following: } } } - + private void HandleCallback(IAsyncResult result) { try @@ -658,38 +658,38 @@ Next, compile and execute the following: { rowText = " row affected."; } - + rowText = rowCount + rowText; - + // You may not interact with the form and its contents // from a different thread, and this callback procedure // is all but guaranteed to be running from a different thread - // than the form. Therefore you cannot simply call code that + // than the form. Therefore you cannot simply call code that // displays the results, like this: // DisplayResults(rowText) - + // Instead, you must call the procedure from the form's thread. // One simple way to accomplish this is to call the Invoke // method of the form, which calls the delegate you supply - // from the form's thread. + // from the form's thread. DisplayInfoDelegate del = new DisplayInfoDelegate(DisplayResults); this.Invoke(del, rowText); - + } catch (Exception ex) { - // Because you are now running code in a separate thread, + // Because you are now running code in a separate thread, // if you do not handle the exception here, none of your other - // code catches the exception. Because none of + // code catches the exception. Because none of // your code is on the call stack in this thread, there is nothing - // higher up the stack to catch the exception if you do not - // handle it here. You can either log the exception or - // invoke a delegate (as in the non-error case in this + // higher up the stack to catch the exception if you do not + // handle it here. You can either log the exception or + // invoke a delegate (as in the non-error case in this // example) to display the error on the form. In no case // can you simply display the error without executing a delegate - // as in the try block here. - - // You can create the delegate instance as you + // as in the try block here. + + // You can create the delegate instance as you // invoke it, like this: this.Invoke(new DisplayInfoDelegate(DisplayStatus), String.Format("Ready(last error: {0}", ex.Message)); @@ -851,7 +851,7 @@ Next, compile and execute the following: // Display the data within the reader. while (reader.Read()) { - // Display all the columns. + // Display all the columns. for (int i = 0; i < reader.FieldCount; i++) { Console.Write("{0}\t", reader.GetValue(i)); @@ -862,8 +862,8 @@ Next, compile and execute the following: private static string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; @@ -953,54 +953,54 @@ Next, compile and execute the following: using System; using System.Data; using Microsoft.Data.SqlClient; - + class Class1 { static void Main() { // This example is not terribly useful, but it proves a point. - // The WAITFOR statement simply adds enough time to prove the + // The WAITFOR statement simply adds enough time to prove the // asynchronous nature of the command. string commandText = "WAITFOR DELAY '00:00:03';" + "SELECT ProductID, Name FROM Production.Product WHERE ListPrice < 100"; - + RunCommandAsynchronously(commandText, GetConnectionString()); - + Console.WriteLine("Press ENTER to continue."); Console.ReadLine(); } - + private static void RunCommandAsynchronously(string commandText, string connectionString) { // Given command text and connection string, asynchronously execute // the specified command against the connection. For this example, - // the code displays an indicator as it is working, verifying the - // asynchronous behavior. + // the code displays an indicator as it is working, verifying the + // asynchronous behavior. try { // The code does not need to handle closing the connection explicitly-- // the use of the CommandBehavior.CloseConnection option takes care - // of that for you. + // of that for you. SqlConnection connection = new SqlConnection(connectionString); SqlCommand command = new SqlCommand(commandText, connection); - + connection.Open(); IAsyncResult result = command.BeginExecuteReader(CommandBehavior.CloseConnection); - + // Although it is not necessary, the following code - // displays a counter in the console window, indicating that - // the main thread is not blocked while awaiting the command + // displays a counter in the console window, indicating that + // the main thread is not blocked while awaiting the command // results. int count = 0; while (!result.IsCompleted) { Console.WriteLine("Waiting ({0})", count++); // Wait for 1/10 second, so the counter - // does not consume all available resources + // does not consume all available resources // on the main thread. System.Threading.Thread.Sleep(100); } - + using (SqlDataReader reader = command.EndExecuteReader(result)) { DisplayResults(reader); @@ -1021,26 +1021,26 @@ Next, compile and execute the following: Console.WriteLine("Error: {0}", ex.Message); } } - + private static void DisplayResults(SqlDataReader reader) { // Display the data within the reader. while (reader.Read()) { - // Display all the columns. + // Display all the columns. for (int i = 0; i < reader.FieldCount; i++) { Console.Write("{0}\t", reader.GetValue(i)); } - + Console.WriteLine(); } } - + private static string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; } @@ -1138,54 +1138,54 @@ Next, compile and execute the following: using System; using System.Data; using Microsoft.Data.SqlClient; - + class Class1 { static void Main() { // This example is not terribly useful, but it proves a point. - // The WAITFOR statement simply adds enough time to prove the + // The WAITFOR statement simply adds enough time to prove the // asynchronous nature of the command. string commandText = "WAITFOR DELAY '00:00:03';" + "SELECT ProductID, Name FROM Production.Product WHERE ListPrice < 100"; - + RunCommandAsynchronously(commandText, GetConnectionString()); - + Console.WriteLine("Press ENTER to continue."); Console.ReadLine(); } - + private static void RunCommandAsynchronously(string commandText, string connectionString) { // Given command text and connection string, asynchronously execute // the specified command against the connection. For this example, - // the code displays an indicator as it is working, verifying the - // asynchronous behavior. + // the code displays an indicator as it is working, verifying the + // asynchronous behavior. try { // The code does not need to handle closing the connection explicitly-- // the use of the CommandBehavior.CloseConnection option takes care - // of that for you. + // of that for you. SqlConnection connection = new SqlConnection(connectionString); SqlCommand command = new SqlCommand(commandText, connection); - + connection.Open(); IAsyncResult result = command.BeginExecuteReader(CommandBehavior.CloseConnection); - + // Although it is not necessary, the following code - // displays a counter in the console window, indicating that - // the main thread is not blocked while awaiting the command + // displays a counter in the console window, indicating that + // the main thread is not blocked while awaiting the command // results. int count = 0; while (!result.IsCompleted) { Console.WriteLine("Waiting ({0})", count++); // Wait for 1/10 second, so the counter - // does not consume all available resources + // does not consume all available resources // on the main thread. System.Threading.Thread.Sleep(100); } - + using (SqlDataReader reader = command.EndExecuteReader(result)) { DisplayResults(reader); @@ -1206,26 +1206,26 @@ Next, compile and execute the following: Console.WriteLine("Error: {0}", ex.Message); } } - + private static void DisplayResults(SqlDataReader reader) { // Display the data within the reader. while (reader.Read()) { - // Display all the columns. + // Display all the columns. for (int i = 0; i < reader.FieldCount; i++) { Console.Write("{0}\t", reader.GetValue(i)); } - + Console.WriteLine(); } } - + private static string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; } @@ -1335,7 +1335,7 @@ Next, compile and execute the following: using System.Text; using System.Windows.Forms; using Microsoft.Data.SqlClient; - + namespace Microsoft.AdoDotNet.CodeSamples { public partial class Form1 : Form @@ -1344,28 +1344,28 @@ Next, compile and execute the following: { InitializeComponent(); } - - // Hook up the form's Load event handler (you can double-click on - // the form's design surface in Visual Studio), and then add + + // Hook up the form's Load event handler (you can double-click on + // the form's design surface in Visual Studio), and then add // this code to the form's class: // You need this delegate in order to fill the grid from // a thread other than the form's thread. See the HandleCallback // procedure for more information. private delegate void FillGridDelegate(SqlDataReader reader); - + // You need this delegate to update the status bar. private delegate void DisplayStatusDelegate(string Text); - + // This flag ensures that the user does not attempt - // to restart the command or close the form while the + // to restart the command or close the form while the // asynchronous command is executing. private bool isExecuting; - + private void DisplayStatus(string Text) { this.label1.Text = Text; } - + private void FillGrid(SqlDataReader reader) { try @@ -1385,7 +1385,7 @@ Next, compile and execute the following: finally { // Closing the reader also closes the connection, - // because this reader was created using the + // because this reader was created using the // CommandBehavior.CloseConnection value. if (reader != null) { @@ -1393,7 +1393,7 @@ Next, compile and execute the following: } } } - + private void HandleCallback(IAsyncResult result) { try @@ -1403,37 +1403,37 @@ Next, compile and execute the following: // of the IAsyncResult parameter. SqlCommand command = (SqlCommand)result.AsyncState; SqlDataReader reader = command.EndExecuteReader(result); - + // You may not interact with the form and its contents // from a different thread, and this callback procedure // is all but guaranteed to be running from a different thread - // than the form. Therefore you cannot simply call code that + // than the form. Therefore you cannot simply call code that // fills the grid, like this: // FillGrid(reader); // Instead, you must call the procedure from the form's thread. // One simple way to accomplish this is to call the Invoke // method of the form, which calls the delegate you supply - // from the form's thread. + // from the form's thread. FillGridDelegate del = new FillGridDelegate(FillGrid); this.Invoke(del, reader); - - // Do not close the reader here, because it is being used in + + // Do not close the reader here, because it is being used in // a separate thread. Instead, have the procedure you have // called close the reader once it is done with it. } catch (Exception ex) { - // Because you are now running code in a separate thread, + // Because you are now running code in a separate thread, // if you do not handle the exception here, none of your other - // code catches the exception. Because there is none of + // code catches the exception. Because there is none of // your code on the call stack in this thread, there is nothing - // higher up the stack to catch the exception if you do not - // handle it here. You can either log the exception or - // invoke a delegate (as in the non-error case in this + // higher up the stack to catch the exception if you do not + // handle it here. You can either log the exception or + // invoke a delegate (as in the non-error case in this // example) to display the error on the form. In no case // can you simply display the error without executing a delegate - // as in the try block here. - // You can create the delegate instance as you + // as in the try block here. + // You can create the delegate instance as you // invoke it, like this: this.Invoke(new DisplayStatusDelegate(DisplayStatus), "Error: " + ex.Message); } @@ -1442,15 +1442,15 @@ Next, compile and execute the following: isExecuting = false; } } - + private string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; } - + private void button1_Click(object sender, System.EventArgs e) { if (isExecuting) @@ -1467,17 +1467,17 @@ Next, compile and execute the following: { DisplayStatus("Connecting..."); connection = new SqlConnection(GetConnectionString()); - // To emulate a long-running query, wait for + // To emulate a long-running query, wait for // a few seconds before retrieving the real data. command = new SqlCommand("WAITFOR DELAY '0:0:5';" + "SELECT ProductID, Name, ListPrice, Weight FROM Production.Product", connection); connection.Open(); - + DisplayStatus("Executing..."); isExecuting = true; - // Although it is not required that you pass the - // SqlCommand object as the second parameter in the + // Although it is not required that you pass the + // SqlCommand object as the second parameter in the // BeginExecuteReader call, doing so makes it easier // to call EndExecuteReader in the callback procedure. AsyncCallback callback = new AsyncCallback(HandleCallback); @@ -1494,13 +1494,13 @@ Next, compile and execute the following: } } } - + private void Form1_Load(object sender, System.EventArgs e) { this.button1.Click += new System.EventHandler(this.button1_Click); this.FormClosing += new FormClosingEventHandler(Form1_FormClosing); } - + void Form1_FormClosing(object sender, FormClosingEventArgs e) { if (isExecuting) @@ -1605,58 +1605,58 @@ Next, compile and execute the following: using System.Data; using Microsoft.Data.SqlClient; using System.Xml; - + class Class1 { static void Main() { // This example is not terribly effective, but it proves a point. - // The WAITFOR statement simply adds enough time to prove the + // The WAITFOR statement simply adds enough time to prove the // asynchronous nature of the command. string commandText = "WAITFOR DELAY '00:00:03';" + "SELECT Name, ListPrice FROM Production.Product " + "WHERE ListPrice < 100 " + "FOR XML AUTO, XMLDATA"; - + RunCommandAsynchronously(commandText, GetConnectionString()); - + Console.WriteLine("Press ENTER to continue."); Console.ReadLine(); } - + private static void RunCommandAsynchronously(string commandText, string connectionString) { // Given command text and connection string, asynchronously execute // the specified command against the connection. For this example, - // the code displays an indicator as it is working, verifying the - // asynchronous behavior. + // the code displays an indicator as it is working, verifying the + // asynchronous behavior. using (SqlConnection connection = new SqlConnection(connectionString)) { SqlCommand command = new SqlCommand(commandText, connection); - + connection.Open(); IAsyncResult result = command.BeginExecuteXmlReader(); - + // Although it is not necessary, the following procedure - // displays a counter in the console window, indicating that - // the main thread is not blocked while awaiting the command + // displays a counter in the console window, indicating that + // the main thread is not blocked while awaiting the command // results. int count = 0; while (!result.IsCompleted) { Console.WriteLine("Waiting ({0})", count++); // Wait for 1/10 second, so the counter - // does not consume all available resources + // does not consume all available resources // on the main thread. System.Threading.Thread.Sleep(100); } - + XmlReader reader = command.EndExecuteXmlReader(result); DisplayProductInfo(reader); } } - + private static void DisplayProductInfo(XmlReader reader) { // Display the data within the reader. @@ -1670,11 +1670,11 @@ Next, compile and execute the following: } } } - + private static string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; } @@ -1788,49 +1788,49 @@ Next, compile and execute the following: using System.Windows.Forms; using System.Xml; using Microsoft.Data.SqlClient; - + namespace Microsoft.AdoDotNet.CodeSamples { public partial class Form1 : Form { - // Hook up the form's Load event handler and then add + // Hook up the form's Load event handler and then add // this code to the form's class: // You need these delegates in order to display text from a thread // other than the form's thread. See the HandleCallback // procedure for more information. private delegate void DisplayInfoDelegate(string Text); private delegate void DisplayReaderDelegate(XmlReader reader); - + private bool isExecuting; - - // This example maintains the connection object + + // This example maintains the connection object // externally, so that it is available for closing. private SqlConnection connection; - + public Form1() { InitializeComponent(); } - + private string GetConnectionString() { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. + // To avoid storing the connection string in your code, + // you can retrieve it from a configuration file. return "Data Source=(local);Integrated Security=true;" + "Initial Catalog=AdventureWorks"; } - + private void DisplayStatus(string Text) { this.label1.Text = Text; } - + private void ClearProductInfo() { // Clear the list box. this.listBox1.Items.Clear(); } - + private void DisplayProductInfo(XmlReader reader) { // Display the data within the reader. @@ -1845,7 +1845,7 @@ Next, compile and execute the following: } DisplayStatus("Ready"); } - + private void Form1_FormClosing(object sender, System.Windows.Forms.FormClosingEventArgs e) { @@ -1856,7 +1856,7 @@ Next, compile and execute the following: e.Cancel = true; } } - + private void button1_Click(object sender, System.EventArgs e) { if (isExecuting) @@ -1873,27 +1873,27 @@ Next, compile and execute the following: ClearProductInfo(); DisplayStatus("Connecting..."); connection = new SqlConnection(GetConnectionString()); - - // To emulate a long-running query, wait for + + // To emulate a long-running query, wait for // a few seconds before working with the data. string commandText = "WAITFOR DELAY '00:00:03';" + "SELECT Name, ListPrice FROM Production.Product " + "WHERE ListPrice < 100 " + "FOR XML AUTO, XMLDATA"; - + command = new SqlCommand(commandText, connection); connection.Open(); - + DisplayStatus("Executing..."); isExecuting = true; - // Although it is not required that you pass the - // SqlCommand object as the second parameter in the + // Although it is not required that you pass the + // SqlCommand object as the second parameter in the // BeginExecuteXmlReader call, doing so makes it easier // to call EndExecuteXmlReader in the callback procedure. AsyncCallback callback = new AsyncCallback(HandleCallback); command.BeginExecuteXmlReader(callback, command); - + } catch (Exception ex) { @@ -1906,7 +1906,7 @@ Next, compile and execute the following: } } } - + private void HandleCallback(IAsyncResult result) { try @@ -1916,34 +1916,34 @@ Next, compile and execute the following: // of the IAsyncResult parameter. SqlCommand command = (SqlCommand)result.AsyncState; XmlReader reader = command.EndExecuteXmlReader(result); - + // You may not interact with the form and its contents // from a different thread, and this callback procedure // is all but guaranteed to be running from a different thread - // than the form. - + // than the form. + // Instead, you must call the procedure from the form's thread. // One simple way to accomplish this is to call the Invoke // method of the form, which calls the delegate you supply - // from the form's thread. + // from the form's thread. DisplayReaderDelegate del = new DisplayReaderDelegate(DisplayProductInfo); this.Invoke(del, reader); - + } catch (Exception ex) { - // Because you are now running code in a separate thread, + // Because you are now running code in a separate thread, // if you do not handle the exception here, none of your other - // code catches the exception. Because none of + // code catches the exception. Because none of // your code is on the call stack in this thread, there is nothing - // higher up the stack to catch the exception if you do not - // handle it here. You can either log the exception or - // invoke a delegate (as in the non-error case in this + // higher up the stack to catch the exception if you do not + // handle it here. You can either log the exception or + // invoke a delegate (as in the non-error case in this // example) to display the error on the form. In no case // can you simply display the error without executing a delegate - // as in the try block here. - - // You can create the delegate instance as you + // as in the try block here. + + // You can create the delegate instance as you // invoke it, like this: this.Invoke(new DisplayInfoDelegate(DisplayStatus), String.Format("Ready(last error: {0}", ex.Message)); @@ -1957,7 +1957,7 @@ Next, compile and execute the following: } } } - + private void Form1_Load(object sender, System.EventArgs e) { this.button1.Click += new System.EventHandler(this.button1_Click); @@ -2031,22 +2031,22 @@ Next, compile and execute the following: using System.Data; using System.Threading; using Microsoft.Data.SqlClient; - + class Program { private static SqlCommand m_rCommand; - + public static SqlCommand Command { get { return m_rCommand; } set { m_rCommand = value; } } - + public static void Thread_Cancel() { Command.Cancel(); } - + static void Main() { string connectionString = GetConnectionString(); @@ -2055,7 +2055,7 @@ Next, compile and execute the following: using (SqlConnection connection = new SqlConnection(connectionString)) { connection.Open(); - + Command = connection.CreateCommand(); Command.CommandText = "DROP TABLE TestCancel"; try @@ -2063,19 +2063,19 @@ Next, compile and execute the following: Command.ExecuteNonQuery(); } catch { } - + Command.CommandText = "CREATE TABLE TestCancel(co1 int, co2 char(10))"; Command.ExecuteNonQuery(); Command.CommandText = "INSERT INTO TestCancel VALUES (1, '1')"; Command.ExecuteNonQuery(); - + Command.CommandText = "SELECT * FROM TestCancel"; SqlDataReader reader = Command.ExecuteReader(); - + Thread rThread2 = new Thread(new ThreadStart(Thread_Cancel)); rThread2.Start(); rThread2.Join(); - + reader.Read(); System.Console.WriteLine(reader.FieldCount); reader.Close(); @@ -2088,7 +2088,7 @@ Next, compile and execute the following: } static private string GetConnectionString() { - // To avoid storing the connection string in your code, + // To avoid storing the connection string in your code, // you can retrieve it from a configuration file. return "Data Source=(local);Initial Catalog=AdventureWorks;" + "Integrated Security=SSPI"; @@ -2528,7 +2528,7 @@ If the option is enabled and a parameter with Direction Output or InputOutput is Although the returns no rows, any output parameters or return values mapped to parameters are populated with data. - For UPDATE, INSERT, and DELETE statements, the return value is the number of rows affected by the command. For all other types of statements, the return value is -1. When a trigger exists on a table being inserted or updated, the return value includes the number of rows affected by both the insert or update operation and the number of rows affected by the trigger or triggers. When SET NOCOUNT ON is set on the connection (before or as part of executing the command, or as part of a trigger initiated by the execution of the command) the rows affected by individual statements stop contributing to the count of rows affected that is returned by this method. If no statements are detected that contribute to the count, the return value is -1. If a rollback occurs, the return value is also -1. + For UPDATE, INSERT, and DELETE statements, the return value is the number of rows affected by the command. For all other types of statements, the return value is -1. When a trigger exists on a table being inserted or updated, the return value includes the number of rows affected by both the insert or update operation and the number of rows affected by the trigger or triggers. When SET NOCOUNT ON is set on the connection (before or as part of executing the command, or as part of a trigger initiated by the execution of the command) the rows affected by individual statements stop contributing to the count of rows affected that is returned by this method. If no statements are detected that contribute to the count, the return value is -1. If a rollback occurs, the return value is also -1. @@ -2540,7 +2540,7 @@ If the option is enabled and a parameter with Direction Output or InputOutput is using System; using System.Data; using Microsoft.Data.SqlClient; - + namespace SqlCommandCS { class Program @@ -2682,7 +2682,7 @@ If you use or , and then executes it by passing a string that is a Transact-SQL SELECT statement, and a string to use to connect to the data source. -[!code-csharp[SqlCommand_ExecuteReader](~/../sqlclient/doc/samples/SqlCommand_ExecuteReader.cs)] +[!code-csharp[SqlCommand_ExecuteReader](~/../sqlclient/doc/samples/SqlCommand_ExecuteReader.cs#1)] ]]> @@ -2750,45 +2750,9 @@ If you use or , and then executes it by passing a string that is a Transact-SQL SELECT statement, and a string to use to connect to the data source. is set to . -[!code-csharp[SqlCommand_ExecuteReader2](~/../sqlclient/doc/samples/SqlCommand_ExecuteReader2.cs#1)] +[!code-csharp[SqlCommand_ExecuteReader2#1](~/../sqlclient/doc/samples/SqlCommand_ExecuteReader2.cs#1)] ]]> - - - The following example creates a , and then executes it by passing a string that is a Transact-SQL SELECT statement, and a string to use to connect to the data source. is set to . - - - - using System; - using System.Data; - using Microsoft.Data.SqlClient; - - class Program - { - static void Main() - { - string str = "Data Source=(local);Initial Catalog=Northwind;" - + "Integrated Security=SSPI"; - string qs = "SELECT OrderID, CustomerID FROM dbo.Orders;"; - CreateCommand(qs, str); - } - - private static void CreateCommand(string queryString, string connectionString) - { - using (SqlConnection connection = new SqlConnection(connectionString)) - { - SqlCommand command = new SqlCommand(queryString, connection); - connection.Open(); - SqlDataReader reader = command.ExecuteReader(CommandBehavior.CloseConnection); - while (reader.Read()) - { - Console.WriteLine(String.Format("{0}", reader[0])); - } - } - } - } - - @@ -3076,7 +3040,7 @@ For more information about asynchronous programming in the .NET Framework Data P using System; using System.Data; using Microsoft.Data.SqlClient; - + public class Sample { public void CreateSqlCommand(string queryString, SqlConnection connection) @@ -3218,7 +3182,7 @@ For more information about asynchronous programming in the .NET Framework Data P using System; using System.Data; using Microsoft.Data.SqlClient; - + private static void CreateXMLReader(string queryString, string connectionString) { using (SqlConnection connection = new SqlConnection(connectionString)) @@ -3474,9 +3438,9 @@ The blocking connection simulates a situation like a command still running in th ### How to use with legacy asynchronous commands Besides assigning the provider to the command and executing the command, it's possible to run it directly using the following methods: -- +- - -- +- [!code-csharp[SqlConfigurableRetryLogic_SqlCommand#4](~/../sqlclient/doc/samples/SqlConfigurableRetryLogic_SqlCommand.cs#4)] @@ -3526,67 +3490,6 @@ The following example demonstrates how to create a - - - The following example demonstrates how to create a and add parameters to the . - - - - using System; - using System.Data; - using Microsoft.Data.SqlClient; - - class Program - { - static void Main() - { - string connectionString = GetConnectionString(); - string demo = @"<StoreSurvey xmlns=""http://schemas.microsoft.com/sqlserver/2004/07/adventure-works/StoreSurvey""><AnnualSales>1500000</AnnualSales><AnnualRevenue>150000</AnnualRevenue><BankName>Primary International</BankName><BusinessType>OS</BusinessType><YearOpened>1974</YearOpened><Specialty>Road</Specialty><SquareFeet<38000</SquareFeet><Brands>3</Brands><Internet>DSL</Internet><NumberEmployees>40</NumberEmployees></StoreSurvey>"; - Int32 id = 3; - UpdateDemographics(id, demo, connectionString); - Console.ReadLine(); - } - private static void UpdateDemographics(Int32 customerID, - string demoXml, string connectionString) - { - // Update the demographics for a store, which is stored - // in an xml column. - string commandText = "UPDATE Sales.Store SET Demographics = @demographics " - + "WHERE CustomerID = @ID;"; - - using (SqlConnection connection = new SqlConnection(connectionString)) - { - SqlCommand command = new SqlCommand(commandText, connection); - command.Parameters.Add("@ID", SqlDbType.Int); - command.Parameters["@ID"].Value = customerID; - - // Use AddWithValue to assign Demographics. - // SQL Server will implicitly convert strings into XML. - command.Parameters.AddWithValue("@demographics", demoXml); - - try - { - connection.Open(); - Int32 rowsAffected = command.ExecuteNonQuery(); - Console.WriteLine("RowsAffected: {0}", rowsAffected); - } - catch (Exception ex) - { - Console.WriteLine(ex.Message); - } - } - } - - static private string GetConnectionString() - { - // To avoid storing the connection string in your code, - // you can retrieve it from a configuration file. - return "Data Source=(local);Initial Catalog=AdventureWorks;" - + "Integrated Security=SSPI"; - } - } - - @@ -3606,7 +3509,7 @@ If you call an `Execute` method after calling . -For vector data types, the property is ignored. The size of the vector is inferred from the of type . +For vector data types, the property is ignored. The size of the vector is inferred from the of type . Prior to Visual Studio 2010, threw an exception. Beginning in Visual Studio 2010, this method does not throw an exception. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml index f0eddd56fa..7a9af8b6b4 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml @@ -262,8 +262,15 @@ The following example creates a and a This property is mutually exclusive with the - - property, among others. + , + , + and + + properties. Setting this property when + + is already set throws + , because SSPI is an + alternative to token-based authentication. @@ -361,7 +368,13 @@ The following example creates a and a This property is mutually exclusive with the - property, among others. + and + + properties, among others. Setting this property when + + is already set throws + , because SSPI is an + alternative to token-based authentication. @@ -827,10 +840,17 @@ The following example creates a an - Empties the connection pool. + Empties all connection pools. - resets (or empties) the connection pool. If there are connections in use at the time of the call, they are marked appropriately and will be discarded (instead of being returned to the pool) when is called on them. + resets (or empties) all connection pools. If there are connections in use at the time of the call, they are marked appropriately and will be discarded (instead of being returned to a pool) when is called on them. + +> [!CAUTION] +> Clearing the pool is an expensive operation and should only be used if required. This operation may negatively interfere with pool warmup and generate high connection churn as the warmup operation continually opens new connections to attempt to reach min pool size. This situation is especially likely if clear is called in a tight loop. + +]]> @@ -841,7 +861,14 @@ The following example creates a an Empties the connection pool associated with the specified connection. - clears the connection pool that is associated with the . If additional connections associated with are in use at the time of the call, they are marked appropriately and are discarded (instead of being returned to the pool) when is called on them. + clears the connection pool that is associated with the `connection`. If additional connections associated with `connection` are in use at the time of the call, they are marked appropriately and are discarded (instead of being returned to the pool) when is called on them. + +> [!CAUTION] +> Clearing the pool is an expensive operation and should only be used if required. This operation may negatively interfere with pool warmup and generate high connection churn as the warmup operation continually opens new connections to attempt to reach min pool size. This situation is especially likely if clear is called in a tight loop. + +]]> @@ -989,7 +1016,7 @@ The following table lists the valid names for keyword values within the
If the value of this key is "", then **Initial Catalog** must be present, and its value must not be "".

The server name can be 128 characters or less.

If you specify a failover partner but the failover partner server is not configured for database mirroring and the primary server (specified with the Server keyword) is not available, then the connection will fail.

If you specify a failover partner and the primary server is not configured for database mirroring, the connection to the primary server (specified with the Server keyword) will succeed if the primary server is available.| |Failover Partner SPN

-or-

FailoverPartnerSPN|N/A|The SPN for the failover partner. The default value is an empty string, which causes SqlClient to use the default, driver-generated SPN.

(Only available in v5.0+)| |Host Name In Certificate

-or-

HostNameInCertificate|N/A|The host name to use when validating the server certificate. When not specified, the server name from the Data Source is used for certificate validation.

(Only available in v5.0+)| -|Server Certificate

-or-

ServerCertificate|N/A|The path to a certificate file to match against the SQL Server TLS/SSL certificate. The accepted certificate formats are PEM, DER, and CER. If specified, the SQL Server certificate is checked by verifying if the ServerCertificate provided is an exact match.

(Only available in v5.1+)| +|Server Certificate

-or-

ServerCertificate|N/A|The path to a certificate file to match against the SQL Server TLS/SSL certificate. The accepted certificate formats are PEM, DER, and CER. If specified, the SQL Server certificate is checked by verifying if the ServerCertificate provided is an exact match.

When specified, this comparison is always performed, including when the certificate already passes the usual chain-and-name validation. If the presented certificate does not match, or the file cannot be loaded or parsed, the TLS handshake fails; a configured `ServerCertificate` is never silently ignored.

Certificate validation itself can be disabled by `TrustServerCertificate=true` (except with `Encrypt=strict`, where validation is always performed). When validation is disabled, `ServerCertificate` is not consulted.

(Only available in v5.1+)| |Initial Catalog

-or-

Database|N/A|The name of the database.

The database name can be 128 characters or less.| |Integrated Security

-or-

Trusted_Connection|'false'|When `false`, User ID and Password are specified in the connection. When `true`, the current Windows account credentials are used for authentication.

Recognized values are `true`, `false`, `yes`, `no`, and `sspi` (strongly recommended), which is equivalent to `true`.

If User ID and Password are specified and Integrated Security is set to true, the User ID and Password will be ignored and Integrated Security will be used.

is a more secure way to specify credentials for a connection that uses SQL Server Authentication (`Integrated Security=false`).| |IP Address Preference

-or-

IPAddressPreference|IPv4First|The IP address family preference when establishing TCP connections. If `Transparent Network IP Resolution` (in .NET Framework) or `Multi Subnet Failover` is set to true, this setting has no effect. Supported values include:

`IPAddressPreference=IPv4First`

`IPAddressPreference=IPv6First`

`IPAddressPreference=UsePlatformDefault`| @@ -1007,7 +1034,7 @@ The following table lists the valid names for keyword values within the
-or-

ServerSPN|N/A|The SPN for the data source. The default value is an empty string, which causes SqlClient to use the default, driver-generated SPN.

(Only available in v5.0+)| |Transaction Binding|Implicit Unbind|Controls connection association with an enlisted `System.Transactions` transaction.

Possible values are:

`Transaction Binding=Implicit Unbind;`

`Transaction Binding=Explicit Unbind;`

Implicit Unbind causes the connection to detach from the transaction when it ends. After detaching, additional requests on the connection are performed in autocommit mode. The `System.Transactions.Transaction.Current` property is not checked when executing requests while the transaction is active. After the transaction has ended, additional requests are performed in autocommit mode.

If the system ends the transaction (in the scope of a using block) before the last command completes, it will throw .

Explicit Unbind causes the connection to remain attached to the transaction until the connection is closed or an explicit `SqlConnection.TransactionEnlist(null)` is called. Beginning in .NET Framework 4.0, changes to Implicit Unbind make Explicit Unbind obsolete. An `InvalidOperationException` is thrown if `Transaction.Current` is not the enlisted transaction or if the enlisted transaction is not active.| -|Transparent Network IP Resolution

-or-

TransparentNetworkIPResolution|See description.|When the value of this key is set to `true`, the application is required to retrieve all IP addresses for a particular DNS entry and attempt to connect with the first one in the list. If the connection is not established within 0.5 seconds, the application will try to connect to all others in parallel. When the first answers, the application will establish the connection with the respondent IP address.

If the `MultiSubnetFailover` key is set to `true`, `TransparentNetworkIPResolution` is ignored.

If the `Failover Partner` key is set, `TransparentNetworkIPResolution` is ignored.

The value of this key must be `true`, `false`, `yes`, or `no`.

A value of `yes` is treated the same as a value of `true`.

A value of `no` is treated the same as a value of `false`.

The default values are as follows:

  • `false` when:

    • Connecting to Azure SQL Database where the data source ends with:

      • .database.chinacloudapi.cn
      • .database.usgovcloudapi.net
      • .database.cloudapi.de
      • .database.windows.net
      • .database.fabric.microsoft.com
    • `Authentication` is 'Active Directory Password' or 'Active Directory Integrated'
  • `true` in all other cases.
| +|Transparent Network IP Resolution

-or-

TransparentNetworkIPResolution|See description.|**Deprecated.** Use `Multi Subnet Failover` instead.

On .NET Framework, when the value of this key is set to `true`, the driver runs multiple connect rounds across the DNS-resolved IP addresses, with progressively larger per-attempt timeouts and a 500 ms minimum on the sequential-mode attempt, until a connection succeeds or the overall `Connect Timeout` is reached.

If the `MultiSubnetFailover` key is set to `true`, `TransparentNetworkIPResolution` is ignored.

If the `Failover Partner` key is set, `TransparentNetworkIPResolution` is ignored.

On .NET Framework, if `TransparentNetworkIPResolution` isn't specified in the connection string, the driver automatically disables TNIR when the data source is an Azure SQL endpoint (`.database.windows.net`, `.database.cloudapi.de`, `.database.usgovcloudapi.net`, `.database.chinacloudapi.cn`, or `.database.fabric.microsoft.com`), when the `Authentication` key is set to any Microsoft Entra ID method (`Active Directory Password`, `Active Directory Integrated`, `Active Directory Interactive`, `Active Directory Service Principal`, `Active Directory Device Code Flow`, `Active Directory Managed Identity`, `Active Directory MSI`, `Active Directory Default`, or `Active Directory Workload Identity`), or when the or property is set on the . For these automatic conditions, an explicit `TransparentNetworkIPResolution` value bypasses the automatic behavior: `True` enables TNIR, and `False` disables TNIR unconditionally. To restore the automatic behavior, remove the keyword from the connection string.

On .NET (Core, .NET 5+), `TransparentNetworkIPResolution` isn't a recognized connection-string keyword. Setting it (with any value) throws when the driver parses the connection string.

On .NET Framework, the value of this key must be `true`, `false`, `yes`, or `no`.

A value of `yes` is treated the same as a value of `true`.

A value of `no` is treated the same as a value of `false`.| |Trust Server Certificate

-or-

TrustServerCertificate|'false'|When set to `true`, TLS is used to encrypt the channel when bypassing walking the certificate chain to validate trust. If TrustServerCertificate is set to `true` and Encrypt is set to `false`, the channel is not encrypted. Recognized values are `true`, `false`, `yes`, and `no`. For more information, see [Connection String Syntax](https://learn.microsoft.com/sql/connect/ado-net/connection-string-syntax).| |Type System Version|N/A|A string value that indicates the type system the application expects. The functionality available to a client application is dependent on the version of SQL Server and the compatibility level of the database. Explicitly setting the type system version that the client application was written for avoids potential problems that could cause an application to break if a different version of SQL Server is used. **Note:** The type system version cannot be set for common language runtime (CLR) code executing in-process in SQL Server. For more information, see [SQL Server Common Language Runtime Integration](https://learn.microsoft.com/dotnet/framework/data/adonet/sql/sql-server-common-language-runtime-integration).

Possible values are:

`Type System Version=SQL Server 2012;`

`Type System Version=SQL Server 2008;`

`Type System Version=SQL Server 2005;`

`Type System Version=Latest;`

`Type System Version=SQL Server 2012;` specifies that the application will require version 11.0.0.0 of Microsoft.SqlServer.Types.dll. The other `Type System Version` settings will require version 10.0.0.0 of Microsoft.SqlServer.Types.dll.

`Latest` is obsolete and should not be used. `Latest` is equivalent to `Type System Version=SQL Server 2008;`.| |User ID

-or-

UID

-or-

User|N/A|The SQL Server login account. Not recommended. To maintain a high level of security, we strongly recommend that you use the `Integrated Security` or `Trusted_Connection` keywords instead. is a more secure way to specify credentials for a connection that uses SQL Server Authentication.

The user ID must be 128 characters or less.| @@ -1357,12 +1384,28 @@ For more information on working with events, see [Connection Events](https://lea - Returns schema information for the data source of this . For more information about scheme, see SQL Server Schema Collections. + Returns schema information for the data source of this . For more information about schemas, see SQL Server Schema Collections. A that contains schema information. + + + The cancellation token. + + + An asynchronous version of , which returns schema information for the data source of this . For more information about schemas, see SQL Server Schema Collections. + + + A task representing the asynchronous operation. + + + + For more information about asynchronous programming in the .NET Framework Data Provider for SQL Server, see Asynchronous Programming. + + + Specifies the name of the schema to return. @@ -1648,6 +1691,28 @@ For more information on working with events, see [Connection Events](https://lea is specified as null.
+ + + Specifies the name of the schema to return. + + + The cancellation token. + + + An asynchronous version of , which returns schema information for the data source of this using the specified string for the schema name. + + + A task representing the asynchronous operation. + + + + For more information about asynchronous programming in the .NET Framework Data Provider for SQL Server, see Asynchronous Programming. + + + + is specified as null. + + Specifies the name of the schema to return. @@ -1677,6 +1742,35 @@ For more information on working with events, see [Connection Events](https://lea
+ + + Specifies the name of the schema to return. + + + A set of restriction values for the requested schema. + + + The cancellation token. + + + An asynchronous version of , which returns schema information for the data source of this using the specified string for the schema name and the specified string array for the restriction values. + + + A task representing the asynchronous operation. + + + + The parameter can supply n depth of values, which are specified by the restrictions collection for a specific collection. In order to set values on a given restriction, and not set the values of other restrictions, you need to set the preceding restrictions to and then put the appropriate value in for the restriction that you would like to specify a value for. + + + An example of this is the "Tables" collection. If the "Tables" collection has three restrictions--database, owner, and table name--and you want to get back only the tables associated with the owner "Carl", you need to pass in the following values: null, "Carl". If a restriction value is not passed in, the default values are used for that restriction. This is the same mapping as passing in , which is different from passing in an empty string for the parameter value. In that case, the empty string ("") is considered to be the value for the specified parameter. + + + + is specified as null. + + + Occurs when SQL Server returns a warning or informational message. @@ -2140,10 +2234,27 @@ The following sample tries to open a connection to an invalid database to simula An instance. + + The + is set while the + or + + property is already set, or the connection is open. + The SspiContextProvider is a part of the connection pool key. Care should be taken when using this property to ensure the implementation returns a stable identity per resource. + + SSPI is an alternative to token-based authentication, so this property + is mutually exclusive with the + and + + properties. Setting this property when either of those is already set + throws , and setting + either of those while this property is set throws as well. Assigning + clears the property and never throws. + @@ -2184,6 +2295,56 @@ The following sample tries to open a connection to an invalid database to simula + + + Gets or sets the middleware application identity reported to the server for this connection. + + + A value. The default is + . + + + + Set the identity before opening the connection: + + + using Microsoft.Data.SqlClient; + + using SqlConnection connection = new(connectionString) + { + RegisteredApplication = RegisteredApplication.EntityFrameworkCore + }; + connection.Open(); + + + + The connection is opening or open. The identity is reported during login, so it must be set beforehand. + + + + This API is intended for registered applications that reserve an identifier in + . An unregistered identifier may be reported by + casting a value to that type. + + + This value is telemetry. It is supplied entirely by the client, which may report any identifier in range, + so it is not an authenticated identity and must not be used for authorization or any other security + decision. + + + The identity is sent once, during login, so it must be set before the connection is opened. + + + When pooling is enabled the value is reported only while establishing a new physical connection, and it is + not part of the pool key. A connection served from the pool therefore reports the identity of whichever + connection caused that physical connection to be created, and physical connections opened in the background + to satisfy Min Pool Size report + . Applications that mix identities over one + connection string should treat this telemetry as indicative rather than exact, or disable pooling where an + exact attribution is required. + + + Gets a string that identifies the database client. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml index a68c0a323b..ffeb1ad240 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml @@ -979,6 +979,31 @@ The following example converts an existing connection string from using SQL Serv + + + Gets or sets the maximum time, in seconds, that a connection can sit unused (idle) in the connection pool before it is discarded. The default is 300 (5 minutes). + + + The idle timeout for pooled connections, in seconds. + + + + This property corresponds to the "Connection Idle Timeout" key within the connection string. + + + In versions where the AppContext switch Switch.Microsoft.Data.SqlClient.UseLegacyIdleTimeoutBehavior is enabled (the default), the driver preserves historical pooling behavior and does not enforce this setting. Set the switch to to enable idle-timeout enforcement. + + + The driver makes a best effort to discard connections that have remained idle in the pool for longer than this value. The exact point in the connection lifecycle at which the check occurs is an implementation detail and may change over time. This protects callers from receiving connections that may have been silently closed by firewalls, load balancers, or server-side inactivity thresholds. + + + A value of zero (0) disables idle expiration; connections are kept in the pool indefinitely (subject to other expiry rules such as ). + + + Idle timeout operates independently of . Whichever threshold is exceeded first causes the connection to be discarded. + + + Gets or sets the maximum number of connections allowed in the connection pool for this specific connection string. @@ -1305,10 +1330,14 @@ Database = AdventureWorks [!NOTE] -> This property only applies when using Integrated Security mode, otherwise it is ignored. +> When `ServerCertificate` is specified, the certificate presented by the server is always compared against the certificate loaded from this path, including when the presented certificate already passes chain-and-name validation. An exact match satisfies certificate validation. +> +> If the presented certificate does not match, or the file cannot be loaded or parsed, the TLS handshake fails; a configured `ServerCertificate` is never silently ignored. +> +> Certificate validation itself can be disabled by `TrustServerCertificate=true` (except with `Encrypt=strict`, where validation is always performed). When validation is disabled, `ServerCertificate` is not consulted. ]]> @@ -1381,12 +1410,15 @@ This property corresponds to the "ServerSPN" and "Server SPN" keys within the co - When the value of this key is set to , the application is required to retrieve all IP addresses for a particular DNS entry and attempt to connect with the first one in the list. If the connection is not established within 0.5 seconds, the application will try to connect to all others in parallel. When the first answers, the application will establish the connection with the respondent IP address. + On .NET Framework, when the value of this key is set to , the driver runs multiple connect rounds across the DNS-resolved IP addresses, with progressively larger per-attempt timeouts and a 500 ms minimum on the sequential-mode attempt, until a connection succeeds or the overall Connect Timeout is reached. A boolean value. + + This property is obsolete. Use instead. + If the Multi Subnet Failover key is set to true, Transparent Network IP Resolution is ignored. @@ -1394,32 +1426,17 @@ This property corresponds to the "ServerSPN" and "Server SPN" keys within the co If the Failover Partner key is set, Transparent Network IP Resolution is ignored. - The value of this key must be true, false, yes, or no. + On .NET Framework, if TransparentNetworkIPResolution isn't specified in the connection string, the driver automatically disables TNIR when the data source is an Azure SQL endpoint (.database.windows.net, .database.cloudapi.de, .database.usgovcloudapi.net, .database.chinacloudapi.cn, or .database.fabric.microsoft.com), when the Authentication key is set to any Microsoft Entra ID method (Active Directory Password, Active Directory Integrated, Active Directory Interactive, Active Directory Service Principal, Active Directory Device Code Flow, Active Directory Managed Identity, Active Directory MSI, Active Directory Default, or Active Directory Workload Identity), or when the or property is set on the . For these automatic conditions, an explicit TransparentNetworkIPResolution value bypasses the automatic behavior: True enables TNIR, and False disables TNIR unconditionally. To restore the automatic behavior, remove the keyword from the connection string. - A value of yes is treated the same as a value of true. A value of no is treated the same as a value of false. + On .NET (Core, .NET 5+), TransparentNetworkIPResolution isn't a recognized connection-string keyword. Setting it (with any value) throws ArgumentException when the driver parses the connection string. - This key defaults to false when: + On .NET Framework, the value of this key must be true, false, yes, or no. + + + A value of yes is treated the same as a value of true. A value of no is treated the same as a value of false. - - - - Connecting to Azure SQL Database where the data source ends with: - - .database.chinacloudapi.cn - .database.usgovcloudapi.net - .database.cloudapi.de - .database.windows.net - .database.fabric.microsoft.com - - - - - Authentication is 'Active Directory Password' or 'Active Directory Integrated' - - Otherwise it defaults to true. - diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlDataAdapter.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlDataAdapter.xml index 7a2f32a0bf..468c5046a6 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlDataAdapter.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlDataAdapter.xml @@ -72,7 +72,7 @@ The following example uses the , , , , , , , , , , , , , , , , , , , The values in the are moved to the parameter values. - The event is raised. + The event is raised. The command executes. If the command is set to FirstReturnedRecord, the first returned result is placed in the . If there are output parameters, they are placed in the . - The event is raised. + The event is raised. is called. @@ -853,55 +853,55 @@ The following example uses the , , , The values in the are moved to the parameter values. - The event is raised. + The event is raised. The command executes. If the command is set to FirstReturnedRecord, the first returned result is placed in the . If there are output parameters, they are placed in the . - The event is raised. + The event is raised. is called. @@ -966,55 +966,55 @@ The following example uses the , , , method retur - Gets the value of the specified column as a . + Gets the value of the specified column as a . - A object representing the column at the given ordinal. + A object representing the column at the given ordinal. The index passed was outside the range of 0 to - 1 @@ -979,7 +979,7 @@ The method retur An attempt was made to read or access columns in a closed . - The retrieved data is not compatible with the type. + The retrieved data is not compatible with the type. No conversions are performed; therefore, the data retrieved must already be a vector value, or an exception is generated. @@ -1193,7 +1193,7 @@ The method retur There is no data ready to be read (for example, the first hasn't been called, or returned false). Tried to read a previously-read column in sequential mode. There was an asynchronous operation in progress. This applies to all Get* methods when running in sequential mode, as they could be called while reading a stream. - + Trying to read a column that does not exist. @@ -1282,21 +1282,21 @@ The method retur // enough for all the columns. Object[] values = new Object[reader.FieldCount]; int fieldCount = reader.GetValues(values); - + Console.WriteLine("reader.GetValues retrieved {0} columns.", fieldCount); for (int i = 0; i < fieldCount; i++) { Console.WriteLine(values[i]); } - + Console.WriteLine(); - - // Now repeat, using an array that may contain a different + + // Now repeat, using an array that may contain a different // number of columns than the original data. This should work correctly, - // whether the size of the array is larger or smaller than + // whether the size of the array is larger or smaller than // the number of columns. - + // Attempt to retrieve three columns of data. values = new Object[3]; fieldCount = reader.GetValues(values); @@ -1334,7 +1334,7 @@ The method retur There is no data ready to be read (for example, the first hasn't been called, or returned false). Trying to read a previously read column in sequential mode. There was an asynchronous operation in progress. This applies to all Get* methods when running in sequential mode, as they could be called while reading a stream. - + Trying to read a column that does not exist. @@ -1394,7 +1394,7 @@ The method retur using System; using System.Data; using Microsoft.Data.SqlClient; - + class Program { static void Main(string[] args) @@ -1448,7 +1448,7 @@ The method retur There is no data ready to be read (for example, the first hasn't been called, or returned false). Trying to read a previously read column in sequential mode. There was an asynchronous operation in progress. This applies to all Get* methods when running in sequential mode, as they could be called while reading a stream. - + Trying to read a column that does not exist. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml index ae7dbdd931..6559612464 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml @@ -505,7 +505,7 @@ The following example creates multiple instances of diff --git a/dotnet-tools.json b/dotnet-tools.json index 1f59e06063..5c185c01e9 100644 --- a/dotnet-tools.json +++ b/dotnet-tools.json @@ -3,18 +3,25 @@ "isRoot": true, "tools": { "dotnet-coverage": { - "version": "18.3.2", + "version": "18.7.0", "commands": [ "dotnet-coverage" ], "rollForward": false }, "microsoft.dotnet.apicompat.tool": { - "version": "10.0.103", + "version": "10.0.300", "commands": [ "apicompat" ], "rollForward": false + }, + "powershell": { + "version": "7.6.2", + "commands": [ + "pwsh" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/eng/pipelines/Pipelines.csproj b/eng/pipelines/Pipelines.csproj new file mode 100644 index 0000000000..f599c80053 --- /dev/null +++ b/eng/pipelines/Pipelines.csproj @@ -0,0 +1,28 @@ + + + + + net10.0 + true + + + + + + diff --git a/eng/pipelines/ci/kerberos/linux-setup-step.yml b/eng/pipelines/ci/kerberos/linux-setup-step.yml new file mode 100644 index 0000000000..c5de26b3ba --- /dev/null +++ b/eng/pipelines/ci/kerberos/linux-setup-step.yml @@ -0,0 +1,110 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Configures a Linux agent, joins it to the domain, acquires a Kerberos ticket, and verifies SQL +# connectivity before running integration tests. + +parameters: + + - name: kerberosDomain + type: string + + - name: kerberosDomainOU + type: string + + - name: kerberosDomainUser + type: string + + - name: kerberosDomainPassword + type: string + +steps: + + - pwsh: | + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json + foreach ($p in $jdata) { + $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING + $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING + $p.SupportsIntegratedSecurity = $true + } + $jdata | Add-Member -NotePropertyName "KerberosDomainUser" -NotePropertyValue $env:KERBEROS_DOMAIN_USER -Force + $jdata | Add-Member -NotePropertyName "KerberosDomainPassword" -NotePropertyValue $env:KERBEROS_DOMAIN_PASSWORD -Force + $jdata | ConvertTo-Json | Set-Content "config.jsonc" + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: Update test config.jsonc (Kerberos) + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) + REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) + KERBEROS_DOMAIN_USER: ${{ parameters.kerberosDomainUser }} + KERBEROS_DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} + + - bash: | + set -euo pipefail + + DOMAIN="${{ parameters.kerberosDomain }}" + DOMAIN_OU="${{ parameters.kerberosDomainOU }}" + DOMAIN_USER="${{ parameters.kerberosDomainUser }}" + DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') + + echo "Domain: $DOMAIN" + echo "Realm: $DOMAIN_UPPER" + echo "User: $DOMAIN_USER" + echo "OU: $DOMAIN_OU" + + if [ -z "${DOMAIN_PASSWORD:-}" ]; then + echo "##vso[task.logissue type=error]KerberosDomainPassword is empty" + exit 1 + fi + + echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + + sudo apt-get -y update + sudo apt-get install -y dialog apt-utils + sudo apt-get install -y \ + krb5-user samba sssd sssd-tools libnss-sss libpam-sss \ + ntp ntpdate realmd adcli + + CURRENT_HOSTNAME="$(hostname)" + if [ "$CURRENT_HOSTNAME" = "$DOMAIN" ] || [[ "$CURRENT_HOSTNAME" == *".$DOMAIN" ]]; then + echo "Hostname already uses domain suffix '.$DOMAIN': $CURRENT_HOSTNAME" + else + sudo hostnamectl set-hostname "$CURRENT_HOSTNAME.$DOMAIN" + fi + + if ! sudo grep -Fqx "server $DOMAIN" /etc/ntp.conf; then + echo "server $DOMAIN" | sudo tee -a /etc/ntp.conf + fi + sudo systemctl stop ntp + sudo ntpdate "$DOMAIN" + sudo systemctl start ntp + + echo "[libdefaults] + default_realm = $DOMAIN_UPPER + rdns = false" | sudo tee /etc/krb5.conf + + sudo realm discover "$DOMAIN_UPPER" + + echo "$DOMAIN_PASSWORD" | sudo realm join --verbose "$DOMAIN_UPPER" \ + -U "$DOMAIN_USER@$DOMAIN_UPPER" \ + --computer-ou "OU=$DOMAIN_OU" + + realm list + + echo "$DOMAIN_PASSWORD" | kinit "$DOMAIN_USER@$DOMAIN_UPPER" + + klist + sudo ip addr + sudo ip route + displayName: Initialize Kerberos (domain join + kinit) + env: + DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} + + - pwsh: | + Install-Module -Name SqlServer -Force -Confirm:$false + Import-Module SqlServer + Invoke-Sqlcmd -Query "SELECT @@VERSION, @@SERVERNAME" -ConnectionString $env:REMOTE_TCP_CONN_STRING + displayName: Verify SQL connectivity + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) \ No newline at end of file diff --git a/eng/pipelines/ci/kerberos/linux-teardown-step.yml b/eng/pipelines/ci/kerberos/linux-teardown-step.yml new file mode 100644 index 0000000000..df01baffc0 --- /dev/null +++ b/eng/pipelines/ci/kerberos/linux-teardown-step.yml @@ -0,0 +1,46 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This template tears down the Linux Kerberos environment by leaving the Active Directory domain +# and destroying credentials. It should be referenced at the end of any job that called +# linux-setup-step.yml. +# +# All steps use condition: always() so that cleanup runs even when previous +# steps fail. + +parameters: + + # The Active Directory domain to leave (e.g. mydomain.contoso.com). + - name: kerberosDomain + type: string + + # The domain user account used during the join. + - name: kerberosDomainUser + type: string + + # The password for the domain user account. + - name: kerberosDomainPassword + type: string + +steps: + + - bash: | + set -uo pipefail + + DOMAIN="${{ parameters.kerberosDomain }}" + DOMAIN_USER="${{ parameters.kerberosDomainUser }}" + DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') + + # Leave the domain + echo "${DOMAIN_PASSWORD:-}" | sudo realm leave "$DOMAIN_UPPER" --verbose \ + -U "$DOMAIN_USER@$DOMAIN_UPPER" || true + + # Destroy the TGT and credential cache + kdestroy || true + displayName: Clean up Kerberos (domain leave + kdestroy) + condition: always() + env: + DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml new file mode 100644 index 0000000000..0a8efa104c --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml @@ -0,0 +1,137 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Runs the Kerberos integration tests for one OS, runtime, and SNI configuration using the exact +# packages produced by the triggering sqlclient-ci-package pipeline. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: displayName + type: string + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: jobNameSuffix + type: string + + - name: operatingSystem + type: string + values: + - Linux + - Windows + + # The pool VM image to use. The pool named by poolName must provide an image with this name. + # + # NOTE: This value is evaluated at template-expansion (compile) time to select the pool image, so + # it must be a literal and must not contain any runtime expressions (e.g. $(...) macros or + # $[...] runtime expressions). + - name: poolImage + type: string + + - name: poolName + type: string + + - name: runtime + type: string + + - name: useManagedSNI + type: boolean + default: false + +jobs: + - job: kerberos_tests_job_${{ parameters.jobNameSuffix }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 90 + + workspace: + clean: all + + pool: + name: ${{ parameters.poolName }} + demands: + - ImageOverride -equals ${{ parameters.poolImage }} + + steps: + + # Align source with the commit that produced the upstream packages while retaining the + # pipeline definitions from the commit at which this run was queued. + - template: /eng/pipelines/common/steps/align-source-with-upstream-step.yml@self + + - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self + + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + debug: ${{ parameters.debug }} + runtimes: [8.x, 9.x, 10.x] + + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet feeds + + - ${{ if eq(parameters.operatingSystem, 'Windows') }}: + - template: /eng/pipelines/ci/kerberos/windows-setup-step.yml@self + parameters: + useManagedSNI: ${{ parameters.useManagedSNI }} + + - ${{ if eq(parameters.operatingSystem, 'Linux') }}: + - template: /eng/pipelines/ci/kerberos/linux-setup-step.yml@self + parameters: + kerberosDomain: $(KerberosDomain) + kerberosDomainOU: $(KerberosDomainOU) + kerberosDomainUser: $(KerberosDomainUser) + kerberosDomainPassword: $(KerberosDomainPassword) + + # UnitTests and FunctionalTests cover environment-independent SPN, SSPI, and connection + # string behavior in normal CI. Run only ManualTests that use the Kerberos environment. + - task: DotNetCoreCLI@2 + displayName: Run Kerberos Integration Tests + retryCountOnTaskFailure: 2 + inputs: + command: build + projects: build.proj + arguments: >- + --verbosity ${{ parameters.dotnetVerbosity }} + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.runtime }} + -p:TestSet=3 + -p:TestFilters="category!=failing&category!=flaky&category!=interactive&(FullyQualifiedName~KerberosTests|FullyQualifiedName~IntegratedAuthenticationTest|FullyQualifiedName~InstanceNameTest)" + -p:ReferenceType=Package + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=$(sqlClientPackageVersion) + -p:PackageVersionSqlServer=$(sqlServerPackageVersion) + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFormat: VSTest + testResultsFiles: $(Build.SourcesDirectory)/test_results/**/*.trx + mergeTestResults: true + testRunTitle: ${{ parameters.displayName }} + buildConfiguration: ${{ parameters.buildConfiguration }} + + - ${{ if eq(parameters.operatingSystem, 'Linux') }}: + - template: /eng/pipelines/ci/kerberos/linux-teardown-step.yml@self + parameters: + kerberosDomain: $(KerberosDomain) + kerberosDomainUser: $(KerberosDomainUser) + kerberosDomainPassword: $(KerberosDomainPassword) \ No newline at end of file diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml new file mode 100644 index 0000000000..200a99c20d --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml @@ -0,0 +1,84 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Kerberos authentication integration tests for Microsoft.Data.SqlClient. The tests consume the +# packages produced by sqlclient-ci-package and run on domain-connected Windows and Linux agents. +# +# Job breakdown: +# - Windows: net462 with native SNI, plus net8/9/10 with native and managed SNI (7 jobs). +# - Linux: net8/9/10 with managed SNI (3 jobs). + +name: $(date:yyyyMMdd)$(rev:.r) + +# Do not trigger this pipeline for PRs or commits. +trigger: none +pr: none + +# The resource branch filter limits completion triggers to the intended upstream branch. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. +resources: + pipelines: + - pipeline: sqlclient-ci-package + source: sqlclient-ci-package + trigger: + branches: + include: + - internal/release/7.1 + +parameters: + + - name: buildConfiguration + displayName: Test Build Configuration + type: string + default: Release + values: + - Debug + - Release + + - name: debug + displayName: Enable debug output + type: boolean + default: false + + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + +variables: + + # Kerberos environment settings shared by all jobs. KerberosDomainPassword resolves from the + # kv-sqldrivers-shared variable group imported by each OS stage. + - name: KerberosDomain + value: sqldrv.ad + + - name: KerberosDomainOU + value: agents + + - name: KerberosDomainUser + value: agent + + - name: KerberosDomainPassword + value: $(agent-at-sqldrv-ad) + + - name: REMOTE_TCP_CONN_STRING + value: Data Source=tcp:sqldrv-sql22.sqldrv.ad\sql2022;Initial Catalog=Northwind;Integrated Security=true;Encrypt=false;TrustServerCertificate=true + + - name: REMOTE_NP_CONN_STRING + value: Data Source=np:sqldrv-sql22.sqldrv.ad\sql2022;Initial Catalog=Northwind;Integrated Security=true;Encrypt=false;TrustServerCertificate=true + +stages: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml new file mode 100644 index 0000000000..90269805c9 --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml @@ -0,0 +1,107 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Defines separate Windows and Linux Kerberos test stages. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: netFrameworkTestRuntimes + type: object + default: [net462] + + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + +stages: + + - stage: windows + displayName: Windows + dependsOn: [] + variables: + - group: kv-sqldrivers-shared + jobs: + + # .NET Framework uses native SNI. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: false + + # .NET runs with both native and managed SNI. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: false + + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Managed SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: true + + - stage: linux + displayName: Linux + dependsOn: [] + variables: + - group: kv-sqldrivers-shared + jobs: + + # Managed SNI is always used on non-Windows platforms. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Linux : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} + operatingSystem: Linux + poolImage: ADO-UB24 + poolName: ADO-Trusted-Linux-WestUS2 + runtime: ${{ runtime }} diff --git a/eng/pipelines/ci/kerberos/windows-setup-step.yml b/eng/pipelines/ci/kerberos/windows-setup-step.yml new file mode 100644 index 0000000000..9a736f68b9 --- /dev/null +++ b/eng/pipelines/ci/kerberos/windows-setup-step.yml @@ -0,0 +1,52 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Configures a domain-connected Windows agent for Kerberos integration tests. + +parameters: + + - name: useManagedSNI + type: boolean + +steps: + + - pwsh: | + $managedSni = [System.Convert]::ToBoolean($env:MANAGED_SNI) + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json + foreach ($p in $jdata) { + $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING + $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING + $p.SupportsIntegratedSecurity = $true + $p.UseManagedSNIOnWindows = $managedSni + } + $jdata | ConvertTo-Json | Set-Content "config.jsonc" + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: Update test config.jsonc + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) + REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) + MANAGED_SNI: '${{ parameters.useManagedSNI }}' + + - powershell: | + $svc = Get-Service -Name SQLBrowser -ErrorAction SilentlyContinue + if ($null -ne $svc) { + Set-Service -StartupType Automatic SQLBrowser + if ($svc.Status -ne 'Running') { Start-Service SQLBrowser } + Get-Service SQLBrowser | Select-Object Name, StartType, Status + } + displayName: Start SQL Server Browser + + - powershell: | + Set-DtcNetworkSetting -DtcName "Local" ` + -InboundTransactionsEnabled $true ` + -OutboundTransactionsEnabled $true ` + -RemoteClientAccessEnabled $true ` + -Confirm:$false + + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC-EPMAP)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-Out)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-In)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + displayName: Enable Network DTC Access \ No newline at end of file diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml new file mode 100644 index 0000000000..3d74f55e0d --- /dev/null +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml @@ -0,0 +1,178 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This job builds and runs the SqlClient Manual test suite against an Azure SQL Managed Instance. +# The SqlClient NuGet packages produced by the sqlclient-ci-package pipeline are downloaded into the +# local NuGet feed (packages/) and the test project is built in "Package" mode +# (ReferenceType=Package) against the exact package versions produced by that pipeline. +# +# The Managed Instance TCP connection string is provided by the "ADO Test Configuration Properties" +# variable group (SQL_MI_TCP_CONN_STRING). This job runs on the Managed-Instance-pool, whose agents +# have network line-of-sight to the Managed Instance. +# +# This template defines a job named 'managed_instance_tests_job_' that can be depended on by +# downstream jobs. + +parameters: + + # The type of build to produce (Debug or Release). + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # True to enable debugging steps. + - name: debug + type: boolean + + # The job's display name. + - name: displayName + type: string + + # The verbosity level for the dotnet CLI commands. + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The suffix to append to the job name. + - name: jobNameSuffix + type: string + + # The OS to build and run the tests for. + - name: operatingSystem + type: string + values: + - Linux + - Windows + + # The .NET runtime (TFM) to build and run the tests for. + - name: runtime + type: string + + # The timeout, in minutes, for this job. + - name: timeout + type: number + default: 90 + + # True to use managed SNI on Windows. Ignored on non-Windows, where managed SNI is always used. + - name: useManagedSNI + type: boolean + default: false + + # The pool VM image to use. The Managed-Instance-pool must provide an image with this name. + # + # NOTE: This value is evaluated at template-expansion (compile) time to select the pool image, so + # it must be a literal and must not contain any runtime expressions (e.g. $(...) macros or + # $[...] runtime expressions). + - name: poolImage + type: string + +jobs: + - job: managed_instance_tests_job_${{ parameters.jobNameSuffix }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: ${{ parameters.timeout }} + + workspace: + clean: all + + variables: + + # Import the variable group that provides the Managed Instance TCP connection string + # (SQL_MI_TCP_CONN_STRING) and other test configuration. + - group: ADO Test Configuration Properties + + # The Managed-Instance-pool agents have network line-of-sight to the Managed Instance. + pool: + name: Managed-Instance-pool + demands: + - imageOverride -equals ${{ parameters.poolImage }} + + steps: + + # Check out the repo and align the working-tree source with the commit that built the upstream + # sqlclient-ci-package artifacts, so the test projects compile against the matching API surface. + - template: /eng/pipelines/common/steps/align-source-with-upstream-step.yml@self + + # Download the SqlClient driver packages published by the triggering sqlclient-ci-package + # pipeline, stage them into the local NuGet feed, and resolve their exact versions into the + # sqlClient/sqlServer/abstractions/logging/azure PackageVersion variables. + - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self + + # Install the .NET SDK and the runtimes needed to execute the test frameworks. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + debug: ${{ parameters.debug }} + runtimes: [8.x, 9.x, 10.x] + + # Restore dotnet CLI tools (e.g. pwsh, apicompat) before building. + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + # Authenticate with NuGet feeds so that upstream packages (e.g. runtime host packs) can be + # fetched through the ADO Artifacts feed. + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet feeds + + # Write the config.jsonc file pointing the tests at the Managed Instance. Integrated security + # and managed identity are not supported against the Managed Instance, and IsManagedInstance + # opts the test suite into Managed-Instance-specific behavior. + - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml@self + parameters: + debug: ${{ parameters.debug }} + # The Managed Instance connection strings carry their own credentials, so no local SA + # password is used here; pass an empty string for the required saPassword parameter. + saPassword: '' + TCPConnectionString: $(SQL_MI_TCP_CONN_STRING) + # Azure SQL Managed Instance exposes TDS over TCP and does not support Named Pipes. + NPConnectionString: '' + SupportsIntegratedSecurity: false + ManagedIdentitySupported: false + IsManagedInstance: true + UseManagedSNIOnWindows: ${{ parameters.useManagedSNI }} + + # Ensure the TestResults directory exists so publish steps don't fail when tests are skipped + # due to a setup failure. + - pwsh: New-Item -ItemType Directory -Path TestResults -Force | Out-Null + displayName: Create TestResults Directory + + # Gate: record that all setup steps completed successfully. Test steps condition on this + # variable so they are skipped when setup fails, yet remain independent of each other's + # results. + - pwsh: Write-Host '##vso[task.setvariable variable=setupSucceeded]true' + displayName: 'Gate: Mark Setup Succeeded' + + # Run ManualTests sets 1, 2, and 3 against the Managed Instance using the exact upstream + # package versions. UnitTests and FunctionalTests do not exercise the configured instance. + # Set AE is omitted because this pipeline does not provision its separate AE, enclave, and AKV + # resources. build.proj's default filter also excludes failing, flaky, and interactive tests. + - task: DotNetCoreCLI@2 + displayName: Run Managed Instance Tests + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) + inputs: + command: build + projects: build.proj + arguments: >- + --verbosity ${{ parameters.dotnetVerbosity }} + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.runtime }} + -p:TestSet=123 + -p:ReferenceType=Package + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=$(sqlClientPackageVersion) + -p:PackageVersionSqlServer=$(sqlServerPackageVersion) + -p:TestResultsFolderPath=TestResults + retryCountOnTaskFailure: 2 + + - template: /eng/pipelines/common/templates/steps/publish-test-results-step.yml@self + parameters: + debug: ${{ parameters.debug }} + targetFramework: ${{ parameters.runtime }} + operatingSystem: ${{ parameters.operatingSystem }} + buildConfiguration: ${{ parameters.buildConfiguration }} diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml new file mode 100644 index 0000000000..978e91fe12 --- /dev/null +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml @@ -0,0 +1,90 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This pipeline runs the SqlClient Unit, Functional, and Manual test suites against an Azure SQL +# Managed Instance, building the test projects in "Package" mode against the NuGet packages produced +# by the sqlclient-ci-package pipeline. It is triggered by successful runs of that pipeline: +# +# ADO.Net project: +# Triggering pipeline: sqlclient-ci-package +# Pipeline name: sqlclient-ci-managed-instance +# +# This pipeline is registered in the ADO.Net (internal) project ONLY, because it depends on the +# Managed Instance connection strings provided by the "ADO Test Configuration Properties" variable +# group and on the Managed-Instance-pool agents, neither of which exist in the Public project. + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Do not trigger this pipeline for PRs, commits, or schedules. +pr: none +trigger: none + +# The resource branch filter limits completion triggers to the intended upstream branch. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. +resources: + pipelines: + + # Trigger this pipeline when the sqlclient-ci-package pipeline completes successfully. + # + # 'project' is omitted so the resource resolves to the sqlclient-ci-package pipeline in the + # *current* ADO project (ADO.Net), where this pipeline is registered. + # + # IMPORTANT: 'source' is the bare pipeline name with no folder path. This REQUIRES the + # sqlclient-ci-package name to be UNIQUE within the ADO.Net project. Azure DevOps only allows + # omitting the folder when the name is unambiguous; if a second pipeline with this name is ever + # added to the project, this resource will fail to resolve and the folder path must be added. + - pipeline: sqlclient-ci-package + source: sqlclient-ci-package + trigger: + branches: + include: + - internal/release/7.1 + +# Pipeline parameters, visible in the Azure DevOps UI. +parameters: + + # The build configuration to use when building the test projects; defaults to Release. + # + # This should match the mode the driver packages we consume were built in - typically Release - + # but may vary if we are triggered due to a manual build of sqlclient-ci-package. + # + - name: buildConfiguration + displayName: Test Build Configuration + type: string + default: Release + values: + - Debug + - Release + + # True to emit debug information and steps. + - name: debug + displayName: Enable debug output + type: boolean + default: false + + # Dotnet CLI verbosity level. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + +# The stages to run. +stages: + + # Run the Managed Instance tests. The .NET and .NET Framework runtimes are defaulted by the stages + # template, which runs both native and managed SNI for .NET on Windows. + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml new file mode 100644 index 0000000000..2176ff7c87 --- /dev/null +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml @@ -0,0 +1,100 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Defines separate Windows and Linux stages that run ManualTests against Azure SQL Managed Instance +# using the packages produced by sqlclient-ci-package. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + + - name: netFrameworkTestRuntimes + type: object + default: [net462] + +stages: + + - stage: windows + displayName: Windows + dependsOn: [] + jobs: + + # .NET Framework uses native SNI. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + displayName: 'Win : Native SNI : ${{ runtime }}' + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + runtime: ${{ runtime }} + useManagedSNI: false + + # .NET runs with both native and managed SNI. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + runtime: ${{ runtime }} + useManagedSNI: false + + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Managed SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + runtime: ${{ runtime }} + useManagedSNI: true + + - stage: linux + displayName: Linux + dependsOn: [] + jobs: + + # Managed SNI is always used on non-Windows platforms. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Linux : ${{ runtime }}' + jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + operatingSystem: Linux + poolImage: ADO-UB24 + runtime: ${{ runtime }} diff --git a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml new file mode 100644 index 0000000000..5bd826b443 --- /dev/null +++ b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml @@ -0,0 +1,175 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This pipeline builds all packages using the build.proj Pack target with ReferenceType=Package, +# then publishes the resulting .nupkg and .snupkg files as a single pipeline artifact. +# +# It runs daily at 13:00 UTC on GitHub dotnet/SqlClient release/7.1 and at 21:00 UTC on ADO.Net +# dotnet-sqlclient internal/release/7.1. +# +# On internal/release/7.1 the strong-name signing key is downloaded and used to sign assemblies +# during the build. +# +# GOTCHA: This pipeline definition is triggered by GitHub _and_ ADO.NET CI. We distinguish the two +# via branch filters: +# +# - Only the GitHub repo has a 'release/7.1' branch. +# - Only the ADO repo has an 'internal/release/7.1' branch. + +name: $(DayOfYear)$(Rev:rr) + +# Do not trigger this pipeline for PRs or pushes. +pr: none +trigger: none + +# Stagger the GitHub and ADO builds so their downstream completion-triggered pipelines do not start +# at the same time. +schedules: + - cron: '0 13 * * *' + displayName: 7.1 GitHub Daily Package Build (13:00 UTC) + branches: + include: + - release/7.1 + + always: true + + - cron: '0 21 * * *' + displayName: 7.1 ADO Daily Package Build (21:00 UTC) + branches: + include: + - internal/release/7.1 + always: true + +# Pipeline parameters visible in the Azure DevOps UI. +parameters: + + # The agent image to use for the build. This must exist in both the ADO-1ES-Pool and + # ADO-CI-1ES-Pool agent pools. + # + # This keeps a default so that manual runs can pick a Windows agent when that is what we + # want to validate. + - name: poolImage + displayName: Pool Image + type: string + default: ADO-UB24 + values: + - ADO-UB24 + - ADO-Win25 + + # The build configuration to use, either Debug or Release. + - name: buildConfiguration + displayName: Build Configuration + type: string + default: Release + values: + - Debug + - Release + + # True to enable debug steps and output. + - name: debug + displayName: Enable debug output + type: boolean + default: false + + # The verbosity level of dotnet CLI commands. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + +variables: + # Provides 'general_purpose_pool_name', the 1ES pool that hosts most jobs in this project. + - group: sqlclient-pipeline-config-v1 + + # Whether this is an internal (ADO.Net project) or public (Public project) build. + - name: isInternalBuild + value: ${{ eq(variables['System.TeamProject'], 'ADO.Net') }} + + # Signing key argument passed to build.proj. On internal builds this references the secure file + # downloaded by DownloadSecureFile@1; on public builds it expands to empty. + - name: signingKeyArg + ${{ if eq(variables.isInternalBuild, true) }}: + value: -p:SigningKeyPath="$(driverKeyFile.secureFilePath)" + ${{ else }}: + value: '' + +jobs: + - job: build_nuget_packages + displayName: Build NuGet Packages + + pool: + name: $(general_purpose_pool_name) + demands: + - ImageOverride -equals ${{ parameters.poolImage }} + + steps: + + # Emit environment variables if debug is enabled. + - ${{ if eq(parameters.debug, true) }}: + - pwsh: | + Get-ChildItem Env: | Sort-Object Name | Format-Table -AutoSize -Wrap + displayName: '[Debug] Print Environment Variables' + + # Install the .NET SDK from global.json. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + debug: ${{ parameters.debug }} + + # Restore dotnet local tools (pwsh, apicompat, etc.). + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + # Clean any pre-existing .nupkg / .snupkg files from the packages/ directory to ensure we only + # publish packages produced by this run. + - pwsh: | + Write-Host 'Cleaning packages/ directory...' + Remove-Item -Force "$(Build.SourcesDirectory)/packages/*.nupkg" -ErrorAction SilentlyContinue + Remove-Item -Force "$(Build.SourcesDirectory)/packages/*.snupkg" -ErrorAction SilentlyContinue + Write-Host 'Done.' + displayName: Clean Packages Directory + + # On internal builds, download the strong-name signing key. + - ${{ if eq(variables.isInternalBuild, true) }}: + - task: DownloadSecureFile@1 + displayName: Download Driver Signing Key + inputs: + secureFile: netfxKeypair.snk + name: driverKeyFile + + # Run the Pack target via build.proj. + - task: DotNetCoreCLI@2 + displayName: Build & Pack + inputs: + command: build + projects: $(Build.SourcesDirectory)/build.proj + arguments: >- + -t:Pack + -p:Configuration=${{ parameters.buildConfiguration }} + -p:ReferenceType=Package + -p:BuildNumber="$(Build.BuildNumber)" + -p:BuildSuffix=ci + $(signingKeyArg) + --verbosity ${{ parameters.dotnetVerbosity }} + + # List produced packages for diagnostics. + - pwsh: | + Write-Host 'Packages produced:' + Get-ChildItem "$(Build.SourcesDirectory)/packages/*.nupkg" -ErrorAction SilentlyContinue | Format-Table Name, Length -AutoSize -Wrap + Get-ChildItem "$(Build.SourcesDirectory)/packages/*.snupkg" -ErrorAction SilentlyContinue | Format-Table Name, Length -AutoSize -Wrap + displayName: List Packages + + # Publish all .nupkg and .snupkg files from packages/ as a pipeline artifact. + - task: PublishPipelineArtifact@1 + displayName: Publish Packages + inputs: + targetPath: $(Build.SourcesDirectory)/packages + artifactName: SqlClient-Driver-Packages + publishLocation: pipeline diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml new file mode 100644 index 0000000000..d7a064523e --- /dev/null +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml @@ -0,0 +1,267 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This job builds and runs stress tests against the SqlClient NuGet packages produced by the +# sqlclient-ci-package pipeline. The package artifacts are downloaded into the local NuGet feed +# (packages/) and the stress test projects are built in "Package" mode (ReferenceType=Package) +# against the exact package versions produced by that pipeline. +# +# The stress tests are located here: +# +# src/Microsoft.Data.SqlClient/tests/StressTests +# +# This template defines a job named 'stress_tests_job_' that can be depended on by +# downstream jobs. + +parameters: + + # The type of build to produce (Debug or Release) + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # True to enable debugging steps. + - name: debug + type: boolean + + # The prefix to prepend to the job's display name: + # + # [] Run Stress Tests + # + - name: displayNamePrefix + type: string + + # The verbosity level for the dotnet CLI commands. + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The suffix to append to the job name. + - name: jobNameSuffix + type: string + + # When true, test failures produce warnings (SucceededWithIssues) but do not fail the job. + # When false, test failures fail the job. All test steps always run regardless of this setting. + - name: warnOnTestFailure + type: boolean + + # The list of .NET Framework runtimes to test against. + - name: netFrameworkTestRuntimes + type: object + + # The list of .NET runtimes to test against. + - name: netTestRuntimes + type: object + + # The local SQL Server instance's 'sa' password, for use in the config file. + - name: saPassword + type: string + + # The step to run to configure SQL Server. This should configure a local SQL Server instance with + # an 'sa' login using the same password provided by the saPassword parameter. + - name: sqlSetupStep + type: step + + # The name of the pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline root. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. + - name: poolName + type: string + + # The pool VM image to use, which must exist in the specified pool. + - name: poolImage + type: string + +jobs: + - job: stress_tests_job_${{ parameters.jobNameSuffix }} + displayName: '[${{ parameters.displayNamePrefix }}] Run Stress Tests' + + workspace: + clean: all + + variables: + + # Import the variable group that provides SQL Server build properties used by the + # shared configure-sql-server-*-step.yml templates (e.g. x64AliasRegistryPath, + # x86AliasRegistryPath, SQLAliasName, SQLAliasPort). + - group: ADO Build Properties + + # Import the variable group that provides SQL Server test configuration variables. + - group: ADO Test Configuration Properties + + # The top-level project file to build and run. + - name: project + value: $(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/tests/StressTests/SqlClient.Stress.Runner/SqlClient.Stress.Runner.csproj + + # The contents of the config file to use for all tests. We will write this to a JSON file and + # then point to it via the STRESS_CONFIG_FILE environment variable. + - name: configContent + value: | + [ + { + "name": "Azure SQL", + "type": "SqlServer", + "isDefault": true, + "dataSource": "localhost", + "user": "sa", + "password": "${{ parameters.saPassword }}", + "supportsWindowsAuthentication": false, + "isLocal": false, + "disableMultiSubnetFailover": true, + "disableNamedPipes": true, + "encrypt": false + } + ] + + # IMPORTANT: Do NOT name pipeline variables "runArguments", "buildArguments", or + # "testArguments". ADO exposes all pipeline variables as environment variables (uppercased), + # and the dotnet CLI's System.CommandLine reads env vars matching {COMMAND}ARGUMENTS (e.g. + # RUNARGUMENTS, BUILDARGUMENTS) and silently injects their content into the parsed arguments — + # bypassing the "--" separator. This causes app arguments to contain SDK options and triggers + # unintended behavior. + + # Reference-mode arguments shared by build and run. The stress tests are built in "Package" + # mode against the SqlClient packages downloaded from the sqlclient-ci-package pipeline. The + # exact versions are resolved at runtime from the downloaded .nupkg filenames (see the + # "Stage Packages and Resolve Versions" step). + - name: referenceArgs + value: >- + -p:ReferenceType=Package + -p:SqlClientPackageVersion=$(sqlClientPackageVersion) + -p:AzurePackageVersion=$(sqlClientPackageVersion) + + # dotnet CLI options for build. + - name: dotnetBuildOpts + value: >- + --verbosity ${{ parameters.dotnetVerbosity }} + -p:Configuration=${{ parameters.buildConfiguration }} + $(referenceArgs) + + # dotnet run options shared by all test steps (framework is appended per-step). + - name: dotnetRunOpts + value: >- + --no-build + --verbosity ${{ parameters.dotnetVerbosity }} + --configuration ${{ parameters.buildConfiguration }} + $(referenceArgs) + + # Stress test options passed after the "--" separator. + - name: stressTestOpts + value: --assembly SqlClient.Stress.Tests --console + + pool: + name: ${{ parameters.poolName }} + + # Images provided by Azure Pipelines must be selected using 'vmImage'. + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: + vmImage: ${{ parameters.poolImage }} + # Images provided by 1ES must be selected using a demand. + ${{ else }}: + demands: + - imageOverride -equals ${{ parameters.poolImage }} + + steps: + + # Check out the repo and align the working-tree source with the commit that built the upstream + # sqlclient-ci-package artifacts, so the stress projects compile against the matching API + # surface. + - template: /eng/pipelines/common/steps/align-source-with-upstream-step.yml@self + + # Install the .NET SDK and Runtimes. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + runtimes: [8.x, 9.x] + + # Setup the local SQL Server. + - ${{ parameters.sqlSetupStep }} + + # Write the config file. + - task: PowerShell@2 + displayName: Write Config File + inputs: + pwsh: true + targetType: inline + script: | + # Capture the multi-line JSON content into a variable. + $content = @" + $(configContent) + "@ + + # Write the JSON content to the config file. + $content | Out-File -FilePath "config.jsonc" + + # Download the SqlClient driver packages published by the triggering sqlclient-ci-package + # pipeline, stage them into the local NuGet feed, and resolve their exact versions. The + # sqlClientPackageVersion variable is consumed by the referenceArgs variable above. + - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self + + # Authenticate with NuGet feeds so that upstream packages (e.g. runtime host packs) can be + # fetched through the ADO Artifacts feed. + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet feeds + + # Build the project. + - task: DotNetCoreCLI@2 + displayName: Build Project + inputs: + command: build + projects: $(project) + arguments: ${{ variables.dotnetBuildOpts }} + + # Set a flag so test steps can distinguish a build failure from a test failure. + - pwsh: Write-Host "##vso[task.setvariable variable=buildSucceeded]true" + displayName: Set build success flag + + # Run the stress tests for each .NET runtime. + # + # The condition and continueOnError work together to achieve the following behavior: + # + # condition: and(succeededOrFailed(), eq(variables['buildSucceeded'], 'true')) + # - succeededOrFailed() allows the step to run even if a *previous test* step failed, + # ensuring all runtimes are exercised regardless of earlier failures. + # - eq(variables['buildSucceeded'], 'true') gates on the flag set above, so tests are + # skipped entirely if the build or any setup step failed (since there's nothing to run). + # + # continueOnError: ${{ parameters.warnOnTestFailure }} + # - When warnOnTestFailure is true, continueOnError is true: a test failure marks the + # step and job as SucceededWithIssues (orange warning) rather than Failed. + # - When warnOnTestFailure is false, continueOnError is false: a test failure fails the + # job (red), though subsequent runtimes still run due to the condition above. + # + - ${{ each runtime in parameters.netTestRuntimes }}: + - task: DotNetCoreCLI@2 + displayName: Test [${{ runtime }}] + condition: and(succeededOrFailed(), eq(variables['buildSucceeded'], 'true')) + continueOnError: ${{ parameters.warnOnTestFailure }} + env: + STRESS_CONFIG_FILE: config.jsonc + inputs: + command: run + projects: $(project) + arguments: ${{ variables.dotnetRunOpts }} -f ${{ runtime }} -- ${{ variables.stressTestOpts }} + + # Run the stress tests for each .NET Framework runtime. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - task: DotNetCoreCLI@2 + displayName: Test [${{ runtime }}] + condition: and(succeededOrFailed(), eq(variables['buildSucceeded'], 'true')) + continueOnError: ${{ parameters.warnOnTestFailure }} + env: + STRESS_CONFIG_FILE: config.jsonc + inputs: + command: run + projects: $(project) + arguments: ${{ variables.dotnetRunOpts }} -f ${{ runtime }} -- ${{ variables.stressTestOpts }} diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml new file mode 100644 index 0000000000..d46f0d2633 --- /dev/null +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml @@ -0,0 +1,112 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This pipeline runs the stress test suite against the SqlClient projects, building them +# transitively as necessary, triggered by successful runs of the sqlclient-ci-package pipeline: +# +# Public project: +# Triggering pipeline: sqlclient-ci-package +# Pipeline name: sqlclient-ci-stress +# +# ADO.Net project: +# Triggering pipeline: sqlclient-ci-package +# Pipeline name: sqlclient-ci-stress + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Do not trigger this pipeline for PRs, commits, or schedules. +pr: none +trigger: none + +# The resource branch filters limit completion triggers to the intended upstream branches. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. +# +# The pipeline identifiers are displayed in the Azure DevOps UI, so it is helpful if they indicate +# the project, folder, and pipeline name, hence the verbose values below. +# +resources: + pipelines: + + # Trigger this pipeline when the sqlclient-ci-package pipeline completes successfully. + # + # 'project' is omitted so the resource resolves to the sqlclient-ci-package pipeline in the + # *current* ADO project. This scopes the trigger per project automatically: the Public + # registration is triggered only by Public's sqlclient-ci-package, and the ADO.Net registration + # only by ADO.Net's. + # + # IMPORTANT: 'source' is the bare pipeline name with no folder path. This REQUIRES the + # sqlclient-ci-package name to be UNIQUE within BOTH the Public and ADO.Net projects. Azure + # DevOps only allows omitting the folder when the name is unambiguous; if a second pipeline with + # this name is ever added to either project, this resource will fail to resolve and the folder + # path must be added back (which, because the folders differ per project, would then require a + # per-project approach instead of this single shared definition). + - pipeline: sqlclient-ci-package + source: sqlclient-ci-package + trigger: + branches: + include: + - release/7.1 + - internal/release/7.1 + +# Pipeline parameters, visible in the Azure DevOps UI. +parameters: + + # The build configuration to use; defaults to Release. + - name: buildConfiguration + displayName: Build Configuration + type: string + default: Release + values: + - Debug + - Release + + # True to emit debug information and steps. + - name: debug + displayName: Enable debug output + type: boolean + default: false + + # When true, test failures produce warnings (SucceededWithIssues) but do not fail the pipeline. + # When false (default), test failures fail the pipeline. + - name: warnOnTestFailure + displayName: Warn (not fail) on test failure + type: boolean + default: false + + # Dotnet CLI verbosity level. + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + +variables: + - template: /eng/pipelines/libraries/ci-build-variables.yml@self + +# The stages to run. +stages: + + # Generate secrets. We use these for the local SQL Server 'sa' logins. + - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self + parameters: + debug: ${{ parameters.debug }} + poolName: $(general_purpose_pool_name) + poolImage: ADO-UB24 + + # Run the stress tests. + - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml@self + parameters: + poolName: $(general_purpose_pool_name) + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + warnOnTestFailure: ${{ parameters.warnOnTestFailure }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml new file mode 100644 index 0000000000..6f8ce4aa6d --- /dev/null +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml @@ -0,0 +1,147 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This stage builds and runs stress tests against the SqlClient projects, building them transitively +# as necessary. +# +# The stress tests are located here: +# +# src/Microsoft.Data.SqlClient/tests/StressTests +# +# All tests use a localhost SQL Server configured for SQL auth via the 'sa' user and the generated +# password. +# +# This stage depends on the secrets_stage. +# +# This template defines a stage named 'stress_tests_stage' that can be depended on by downstream +# stages. + +parameters: + + # The name of the pool to use for jobs that require customized VM images. + # + # Supplied by the caller so that the pool name flows down from the pipeline root. + - name: poolName + type: string + + # The type of build to produce (Debug or Release) + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # True to enable debugging steps. + - name: debug + type: boolean + + # When true, test failures produce warnings (SucceededWithIssues) but do not fail the job. + # When false, test failures fail the job. All test steps always run regardless of this setting. + - name: warnOnTestFailure + type: boolean + + # The verbosity level for the dotnet CLI commands. + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The list of .NET Framework runtimes to test against. + - name: netFrameworkTestRuntimes + type: object + default: [net462] + + # The list of .NET runtimes to test against. These should include the TFMs that SqlClient ships + # as well as any upcoming runtimes being validated (e.g. net10.0 is tested but not yet shipped). + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + +stages: + - stage: stress_tests_stage + displayName: Run Stress Tests + dependsOn: + - secrets_stage + + variables: + # Bring the SA password from the secrets_stage into scope here. + - name: saPassword + value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] + + jobs: + + # ---------------------------------------------------------------------------------------------- + # Build and test on Linux. + + - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayNamePrefix: Linux + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + warnOnTestFailure: ${{ parameters.warnOnTestFailure }} + jobNameSuffix: linux + # No .NET Framework runtimes on Linux. + netFrameworkTestRuntimes: [] + netTestRuntimes: ${{ parameters.netTestRuntimes }} + saPassword: $(saPassword) + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self + parameters: + saPassword: $(saPassword) + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24-SQL25 + + # ---------------------------------------------------------------------------------------------- + # Build and test on Windows + + - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayNamePrefix: Win + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + warnOnTestFailure: ${{ parameters.warnOnTestFailure }} + jobNameSuffix: windows + # Note that we include the .NET Framework runtimes for test runs on Windows. + netFrameworkTestRuntimes: ${{ parameters.netFrameworkTestRuntimes }} + netTestRuntimes: ${{ parameters.netTestRuntimes }} + saPassword: $(saPassword) + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml@self + parameters: + saPassword: $(saPassword) + # The Windows images include a suitable .NET Framework runtime, so we don't have to install + # one explicitly. + poolName: ${{ parameters.poolName }} + poolImage: ADO-MMS25-SQL25 + + # ---------------------------------------------------------------------------------------------- + # Build and test on macOS. + + - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayNamePrefix: macOS + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + warnOnTestFailure: ${{ parameters.warnOnTestFailure }} + jobNameSuffix: macos + # No .NET Framework runtimes on macOS. + netFrameworkTestRuntimes: [] + netTestRuntimes: ${{ parameters.netTestRuntimes }} + saPassword: $(saPassword) + sqlSetupStep: + template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self + parameters: + saPassword: $(saPassword) + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. + poolName: Azure Pipelines + poolImage: macos-latest diff --git a/eng/pipelines/common/steps/align-source-with-upstream-step.yml b/eng/pipelines/common/steps/align-source-with-upstream-step.yml new file mode 100644 index 0000000000..39249d572d --- /dev/null +++ b/eng/pipelines/common/steps/align-source-with-upstream-step.yml @@ -0,0 +1,63 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This template checks out the repository and aligns the working-tree source with the exact commit +# that produced the artifacts consumed from an upstream (triggering) pipeline resource (e.g. +# sqlclient-ci-package). This keeps the source that gets built and tested in lockstep with the +# upstream artifacts' API surface. +# +# IMPORTANT: Pipeline definitions and @self templates are compiled from the triggered branch tip at +# queue time, but file-based tasks still read scripts from the runtime working tree. After aligning +# the source, this template restores eng/pipelines from the queued commit so compiled pipeline steps +# and their scripts remain in lockstep. +# +# The consuming pipeline MUST declare the upstream pipeline as a resource whose alias matches the +# upstreamPipeline parameter, e.g.: +# +# resources: +# pipelines: +# - pipeline: sqlclient-ci-package +# source: sqlclient-ci-package +# trigger: true + +parameters: + + # The upstream pipeline resource alias whose source commit the working tree should be aligned to. + # This must match a resource declared under resources.pipelines in the consuming pipeline. + - name: upstreamPipeline + type: string + default: sqlclient-ci-package + +steps: + + # Check out the repo with full history so the working tree can be reset to an arbitrary commit. + - checkout: self + clean: true + fetchDepth: 0 + fetchTags: false + persistCredentials: true + + # Align the working-tree source with the commit that built the upstream artifacts, so the projects + # compile against the matching (internal) API surface. When the upstream commit is unavailable + # (e.g. some manual runs), skip this step and use the checked-out source as-is. + - pwsh: | + $ErrorActionPreference = 'Stop' + $sha = "$(resources.pipeline.${{ parameters.upstreamPipeline }}.sourceCommit)" + if ($sha -notmatch '\A[0-9a-fA-F]{40}\z') { + throw "Invalid ${{ parameters.upstreamPipeline }} commit SHA: '$sha'." + } + $pipelineSourceSha = git rev-parse HEAD + if ($LASTEXITCODE -ne 0) { throw "Failed to resolve the queued pipeline commit." } + Write-Host "Fetching ${{ parameters.upstreamPipeline }} commit $sha" + git fetch --no-tags origin $sha + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch upstream commit $sha." } + Write-Host "Aligning source to ${{ parameters.upstreamPipeline }} commit $sha" + git checkout --force $sha + if ($LASTEXITCODE -ne 0) { throw "Failed to check out upstream commit $sha." } + Write-Host "Restoring eng/pipelines from queued pipeline commit $pipelineSourceSha" + git checkout --force $pipelineSourceSha -- eng/pipelines + if ($LASTEXITCODE -ne 0) { throw "Failed to restore eng/pipelines from $pipelineSourceSha." } + displayName: Align Source With Upstream Commit + condition: ne(variables['resources.pipeline.${{ parameters.upstreamPipeline }}.sourceCommit'], '') diff --git a/eng/pipelines/common/steps/download-driver-packages-step.yml b/eng/pipelines/common/steps/download-driver-packages-step.yml new file mode 100644 index 0000000000..807affea13 --- /dev/null +++ b/eng/pipelines/common/steps/download-driver-packages-step.yml @@ -0,0 +1,61 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# This template downloads the SqlClient driver packages published by the sqlclient-ci-package +# pipeline, copies them into the local NuGet feed (packages/), and resolves the exact package +# versions from the .nupkg filenames. Required SqlClient-family packages are validated to share +# one version. The resolved versions are exposed as job-scoped pipeline variables for use by +# downstream build/test steps: +# +# sqlClientPackageVersion SqlClient family +# sqlServerPackageVersion Microsoft.SqlServer.Server +# +# The consuming pipeline MUST declare the triggering pipeline as a resource whose alias matches the +# pipelineResource parameter, e.g.: +# +# resources: +# pipelines: +# - pipeline: sqlclient-ci-package +# source: sqlclient-ci-package +# trigger: true + +parameters: + + # The pipeline resource alias to download the packages from. This must match a resource declared + # under resources.pipelines in the consuming pipeline. + - name: pipelineResource + type: string + default: sqlclient-ci-package + + # The name of the artifact published by the triggering pipeline that contains the .nupkg/.snupkg + # files. + - name: artifactName + type: string + default: SqlClient-Driver-Packages + + # The local NuGet feed directory to copy the downloaded packages into. + - name: feedPath + type: string + default: $(Build.SourcesDirectory)/packages + +steps: + + # Download the SqlClient driver packages published by the triggering pipeline into the pipeline + # workspace. + - download: ${{ parameters.pipelineResource }} + artifact: ${{ parameters.artifactName }} + displayName: Download SqlClient Driver Packages + + # Copy the downloaded packages into the local NuGet feed, validate the SqlClient family version, + # and expose the two independently versioned package units as pipeline variables. + - task: PowerShell@2 + displayName: Stage Packages and Resolve Versions + inputs: + pwsh: true + targetType: filePath + filePath: $(Build.SourcesDirectory)/eng/pipelines/common/steps/download-driver-packages.ps1 + arguments: >- + -FeedPath "${{ parameters.feedPath }}" + -ArtifactDirectory "$(Pipeline.Workspace)/${{ parameters.pipelineResource }}/${{ parameters.artifactName }}" diff --git a/eng/pipelines/common/steps/download-driver-packages.ps1 b/eng/pipelines/common/steps/download-driver-packages.ps1 new file mode 100644 index 0000000000..4392508991 --- /dev/null +++ b/eng/pipelines/common/steps/download-driver-packages.ps1 @@ -0,0 +1,128 @@ +<# +.SYNOPSIS + Stages SqlClient driver packages and exposes their exact versions to Azure Pipelines. + +.DESCRIPTION + Copies the .nupkg and .snupkg files downloaded from an upstream SqlClient package artifact into + a local NuGet feed. It resolves package versions from the .nupkg filenames, verifies that all + required SqlClient-family packages share one version, and emits Azure Pipelines logging commands + that create these job-scoped variables for downstream tasks: + + sqlClientPackageVersion SqlClient family + sqlServerPackageVersion Microsoft.SqlServer.Server + + Every required .nupkg must be present in ArtifactDirectory. Symbol packages are optional. The + script stops on missing required packages, copy failures, and ambiguous filesystem errors. + +.PARAMETER FeedPath + Directory used as the local NuGet feed. The script creates it when it does not exist and + overwrites packages with matching filenames. + +.PARAMETER ArtifactDirectory + Directory containing the .nupkg files downloaded from the upstream pipeline artifact. Optional + .snupkg files in this directory are copied when present. + +.EXAMPLE + ./download-driver-packages.ps1 ` + -FeedPath 'C:\agent\_work\1\s\packages' ` + -ArtifactDirectory 'C:\agent\_work\1\sqlclient-ci-package\SqlClient-Driver-Packages' + + Stages all driver packages and resolves every version from its package filename. + +.OUTPUTS + None. Results are emitted as Azure Pipelines task.setvariable logging commands. + +.NOTES + This script is designed for PowerShell Core in Azure Pipelines. Package filenames must use the + conventional ..nupkg format, and versions must begin with a digit. +#> + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string]$FeedPath, + + [Parameter(Mandatory)] + [string]$ArtifactDirectory +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +New-Item -ItemType Directory -Force -Path $FeedPath | Out-Null + +Copy-Item "$ArtifactDirectory/*.nupkg" $FeedPath -Force +Copy-Item "$ArtifactDirectory/*.snupkg" $FeedPath -Force -ErrorAction SilentlyContinue + +function Resolve-PackageVersion { + <# + .SYNOPSIS + Resolves a package version from a matching .nupkg filename. + #> + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][string]$Pattern, + [Parameter(Mandatory)][string]$PackageName + ) + + $packages = @(Get-ChildItem "$Path/*.nupkg" | Where-Object { $_.Name -match $Pattern }) + + if ($packages.Count -ne 1) { + throw "Expected exactly one $PackageName package in $Path, found $($packages.Count)" + } + + return [regex]::Match($packages[0].Name, $Pattern).Groups[1].Value +} + +# Patterns are anchored so Microsoft.Data.SqlClient does not also match family packages whose IDs +# begin with Microsoft.Data.SqlClient. +$sqlClientFamilyPackages = @( + @{ + Name = 'Microsoft.Data.SqlClient' + Pattern = '^Microsoft\.Data\.SqlClient\.(\d[^\/]*)\.nupkg$' + }, + @{ + Name = 'Microsoft.Data.SqlClient.Extensions.Abstractions' + Pattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Abstractions\.(\d[^\/]*)\.nupkg$' + }, + @{ + Name = 'Microsoft.Data.SqlClient.Internal.Logging' + Pattern = '^Microsoft\.Data\.SqlClient\.Internal\.Logging\.(\d[^\/]*)\.nupkg$' + }, + @{ + Name = 'Microsoft.Data.SqlClient.Extensions.Azure' + Pattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Azure\.(\d[^\/]*)\.nupkg$' + }, + @{ + Name = 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' + Pattern = '^Microsoft\.Data\.SqlClient\.AlwaysEncrypted\.AzureKeyVaultProvider\.(\d[^\/]*)\.nupkg$' + } +) + +$sqlClientPackageVersion = Resolve-PackageVersion ` + $ArtifactDirectory ` + $sqlClientFamilyPackages[0].Pattern ` + $sqlClientFamilyPackages[0].Name + +foreach ($package in $sqlClientFamilyPackages) { + $version = Resolve-PackageVersion $ArtifactDirectory $package.Pattern $package.Name + + if ($version -ne $sqlClientPackageVersion) { + throw "$($package.Name) version $version does not match SqlClient family version $sqlClientPackageVersion" + } + + Write-Host "Validated $($package.Name) version: $version" +} + +$sqlServerPackageVersion = Resolve-PackageVersion ` + $ArtifactDirectory ` + '^Microsoft\.SqlServer\.Server\.(\d[^\/]*)\.nupkg$' ` + 'Microsoft.SqlServer.Server' + +Write-Host "Resolved Microsoft.SqlServer.Server version: $sqlServerPackageVersion" +Write-Host "##vso[task.setvariable variable=sqlClientPackageVersion]$sqlClientPackageVersion" +Write-Host "##vso[task.setvariable variable=sqlServerPackageVersion]$sqlServerPackageVersion" diff --git a/eng/pipelines/common/steps/install-dotnet.yml b/eng/pipelines/common/steps/install-dotnet.yml new file mode 100644 index 0000000000..70179279b3 --- /dev/null +++ b/eng/pipelines/common/steps/install-dotnet.yml @@ -0,0 +1,76 @@ +################################################################################ +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################ + +# This template installs a single .NET SDK and zero or more .NET Runtimes. The +# SDK version is always read from the root global.json file. The Runtimes to +# install, if any, may be provided as an array of version strings. +# +# The UseDotNet@2 task installs the SDK/Runtimes for the agent's native +# architecture, including ARM64. (The historical ARM64 workaround for +# https://github.com/microsoft/azure-pipelines-tasks/issues/20300 is no longer +# needed now that the task correctly detects win-arm64 in v271+.) + +parameters: + + # True to force installation of the x86 build (e.g. for x86 test runs on an + # x64 host). When false, UseDotNet@2 installs for the agent's native + # architecture. + - name: forceX86 + type: boolean + default: false + + # True to emit debug information and steps. + - name: debug + type: boolean + default: false + + # The directory to install to. + - name: installDir + type: string + default: $(Agent.ToolsDirectory)/dotnet + + # The list of .NET Runtimes to install, if any. These must adhere to the + # format expected by the UseDotNet@2 task: + # + # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/use-dotnet-v2 + # + - name: runtimes + type: object + default: [] + +steps: + + # Install the SDK listed in the global.json file. + # + # retryCountOnTaskFailure is set because UseDotNet@2 fails intermittently + # in CI due to transient network/CDN issues when downloading the SDK. + - task: UseDotNet@2 + displayName: Install .NET SDK (global.json) + retryCountOnTaskFailure: 3 + inputs: + installationPath: ${{ parameters.installDir }} + packageType: sdk + useGlobalJson: true + ${{ if parameters.forceX86 }}: + env: + PROCESSOR_ARCHITECTURE: x86 + + # Install the desired Runtimes, if any. + - ${{ each version in parameters.runtimes }}: + - task: UseDotNet@2 + displayName: Install .NET ${{ version }} Runtime + retryCountOnTaskFailure: 3 + inputs: + installationPath: ${{ parameters.installDir }} + packageType: runtime + version: ${{ version }} + ${{ if parameters.forceX86 }}: + env: + PROCESSOR_ARCHITECTURE: x86 + + # Report what was installed. + - pwsh: dotnet --info + displayName: Report installed .NET SDK and Runtimes diff --git a/eng/pipelines/common/steps/restore-dotnet-tools.yml b/eng/pipelines/common/steps/restore-dotnet-tools.yml new file mode 100644 index 0000000000..b213583e82 --- /dev/null +++ b/eng/pipelines/common/steps/restore-dotnet-tools.yml @@ -0,0 +1,14 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Restores dotnet CLI tools defined in dotnet-tools.json. +# This step should be invoked after install-dotnet.yml and before any build +# steps that depend on the restored tools (e.g. pwsh, apicompat). + +steps: + - script: dotnet tool restore + displayName: Restore .NET Tools + workingDirectory: $(Build.SourcesDirectory) diff --git a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml index 5e17b506c1..9b7a88e6cc 100644 --- a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml @@ -19,15 +19,17 @@ parameters: # Reference sibling packages as C# projects. - Project - # The name of Azure Pipelines pool to use. + # The name of the 1ES pool to use. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # - name: poolName type: string - default: $(ci_var_defaultPoolName) - # The name of the Azure Pipelines image to use within the pool. - - name: imageOverride + # The name of the VM image to run on, within the pool. + - name: poolImage type: string - default: ADO-MMS22-SQL19 # The name of the Abstractions pipeline artifact to download when referenceType is 'Package'. - name: abstractionsArtifactsName @@ -61,21 +63,20 @@ parameters: type: stepList default: [] - # The version of the Abstractions package to depend on when referenceType is 'Package'. - - name: abstractionsPackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging, and the + # AKV Provider). They all share this version. + - name: packageVersion type: string - # The version of the Logging package to depend on when referenceType is 'Package'. - - name: loggingPackageVersion + # The version of the SqlServer package to depend on when referenceType is 'Package'. + - name: sqlServerPackageVersion type: string + default: $(sqlServerPackageVersion) - # The version to apply to the SqlClient package. - - name: mdsPackageVersion - type: string - - # The version to apply to the AKV Provider package. - - name: akvPackageVersion + # The name of the SqlServer pipeline artifact to download when referenceType is 'Package'. + - name: sqlServerArtifactsName type: string + default: SqlServer.Artifacts jobs: - job: build_mds_akv_packages_job @@ -84,11 +85,7 @@ jobs: pool: name: ${{parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.imageOverride }} - - msbuild - - variables: - - template: /eng/pipelines/libraries/ci-build-variables.yml@self + - imageOverride -equals ${{ parameters.poolImage }} steps: - ${{ if eq(parameters.debug, true)}}: @@ -116,8 +113,17 @@ jobs: artifactName: ${{ parameters.loggingArtifactsName }} targetPath: $(localFeedPath) + - task: DownloadPipelineArtifact@2 + displayName: Download SqlServer Package Artifacts + inputs: + artifactName: ${{ parameters.sqlServerArtifactsName }} + targetPath: $(localFeedPath) + # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + # Restore dotnet CLI tools (e.g. pwsh, apicompat) before building. + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self # When we're performing a Debug build, we still want to try _compiling_ the # code in Release mode to ensure downstream pipelines don't encounter @@ -130,7 +136,6 @@ jobs: buildConfiguration: Release referenceType: Project build: all - assemblyBuildNumber: $(assemblyBuildNumber) - template: /eng/pipelines/common/templates/steps/ci-project-build-step.yml@self parameters: @@ -139,20 +144,31 @@ jobs: referenceType: ${{ parameters.referenceType }} operatingSystem: Windows build: MDS - assemblyBuildNumber: $(assemblyBuildNumber) - abstractionsPackageVersion: ${{parameters.abstractionsPackageVersion}} - loggingPackageVersion: ${{ parameters.loggingPackageVersion }} + packageVersion: ${{ parameters.packageVersion }} + sqlServerPackageVersion: ${{ parameters.sqlServerPackageVersion }} - - template: /eng/pipelines/common/templates/steps/generate-nuget-package-step.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - displayName: 'Create MDS NuGet Package' - generateSymbolsPackage: true - nuspecPath: 'tools/specs/Microsoft.Data.SqlClient.nuspec' - outputDirectory: $(packagePath) - packageVersion: ${{ parameters.mdsPackageVersion }} - properties: 'AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }};LoggingPackageVersion=${{ parameters.loggingPackageVersion }}' - referenceType: ${{ parameters.referenceType }} + - task: DotNetCoreCLI@2 + displayName: 'Create MDS NuGet Package' + inputs: + command: build + projects: build.proj + arguments: >- + -t:PackSqlClient + -p:PackBuild=false + -p:Configuration=${{ parameters.buildConfiguration }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + + # PackSqlClient outputs to artifacts/Microsoft.Data.SqlClient/-/. + # Downstream steps (local feed copy, AKV pack, artifact publish) all expect packages at + # $(packagePath), so copy them there. + - task: CopyFiles@2 + displayName: Copy MDS Package to Output Directory + inputs: + sourceFolder: $(Build.SourcesDirectory)/artifacts/Microsoft.Data.SqlClient/${{ parameters.referenceType }}-${{ parameters.buildConfiguration }} + contents: 'Microsoft.Data.SqlClient.*' + targetFolder: $(packagePath) # When building in Package mode, the AKV Provider restore needs to find the MDS package # we just built. Copy it to the local NuGet feed so NuGet.config can resolve it. @@ -170,26 +186,32 @@ jobs: buildConfiguration: ${{ parameters.buildConfiguration }} referenceType: ${{ parameters.referenceType }} build: AkvProvider - assemblyBuildNumber: $(assemblyBuildNumber) - abstractionsPackageVersion: ${{parameters.abstractionsPackageVersion}} - loggingPackageVersion: ${{ parameters.loggingPackageVersion }} - mdsPackageVersion: ${{ parameters.mdsPackageVersion }} - akvPackageVersion: ${{ parameters.akvPackageVersion }} + packageVersion: ${{ parameters.packageVersion }} + sqlServerPackageVersion: ${{ parameters.sqlServerPackageVersion }} - - task: MSBuild@1 + - task: DotNetCoreCLI@2 displayName: 'Create AKV Provider NuGet Package' inputs: - solution: build.proj - msbuildArchitecture: x64 - platform: '${{ parameters.platform }}' - configuration: '${{ parameters.buildConfiguration }}' - msbuildArguments: >- + command: build + projects: build.proj + arguments: >- -t:PackAkvProvider + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackBuild=false -p:ReferenceType=${{ parameters.referenceType }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:AkvPackageVersion=${{ parameters.akvPackageVersion }} - -p:PackageOutputPath=$(packagePath) + -p:BuildNumber=$(Build.BuildNumber) + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + + - task: CopyFiles@2 + displayName: Copy AKV Package to Output Folder + inputs: + sourceFolder: artifacts/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/${{ parameters.buildConfiguration }} + contents: | + **/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider*.nupkg + **/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider*.snupkg + targetFolder: $(packagePath) + flattenFolders: true - task: PublishPipelineArtifact@1 displayName: 'Publish Pipeline Artifacts' diff --git a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml index 09c72e9719..0b71bbaa5e 100644 --- a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml @@ -19,13 +19,27 @@ parameters: - name: upload type: boolean + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: publish_code_coverage displayName: Publish Code Coverage pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: # Use a temp directory that is cleaned up after each job runs. This helps @@ -44,7 +58,7 @@ jobs: displayName: '[Debug] List Environment Variables' # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} @@ -112,12 +126,13 @@ jobs: displayName: '[Debug] Show Disk Usage' # Publish the Cobertura XML coverage file as a pipeline artifact for - # debugging purposes. + # debugging purposes. Pipeline artifact names must be unique within a + # build, so include the attempt numbers to allow this job to be rerun. - task: PublishPipelineArtifact@1 displayName: Publish Cobertura XML Artifact inputs: targetPath: $(workingDir)/merge - artifact: Cobertura Merge Results + artifact: Cobertura Merge Results $(System.StageAttempt)-$(System.JobAttempt) # Publish the Cobertura reports to the pipeline to be viewed in the Azure # DevOps pipeline run UI. diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index 76a3bf3c75..86ee3806fd 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -8,19 +8,11 @@ parameters: - name: abstractionsArtifactsName type: string - # The version of the Abstractions package to depend on when referenceType is 'Package'. - - name: abstractionsPackageVersion - type: string - # The name of the Logging pipeline artifact to download when referenceType is 'Package'. - name: loggingArtifactsName type: string default: Logging.Artifacts - # The version of the Logging package to depend on when referenceType is 'Package'. - - name: loggingPackageVersion - type: string - # The configuration properties to set in the config file. # # GOTCHA: The following keys are used in template expressions and must be @@ -30,11 +22,9 @@ parameters: # template expansion. # # EnclaveEnabled - # IsAzureSynapse # IsDNSCachingSupportedCR # IsDNSCachingSupportedTR # ManagedIdentitySupported - # SupportsFileStream # SupportsIntegratedSecurity # TracingEnabled # @@ -66,14 +56,8 @@ parameters: type: boolean default: false - # True if this job will run in a generic Azure Pipelines hosted pool; false to run in a custom 1ES - # pool. - - name: hostedPool - type: boolean - default: false - # The VM image to use, which must exist in the specified pool. - - name: image + - name: poolImage type: string # The display name for this job. @@ -84,10 +68,21 @@ parameters: - name: mdsArtifactsName type: string - # The version of the SqlClient package to depend on when referenceType is 'Package'. - - name: mdsPackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging). They + # all share this version. + - name: packageVersion type: string + # The name of the SqlServer pipeline artifact to download when referenceType is 'Package'. + - name: sqlServerArtifactsName + type: string + default: SqlServer.Artifacts + + # The version of the SqlServer package to depend on when referenceType is 'Package'. + - name: sqlServerPackageVersion + type: string + default: $(sqlServerPackageVersion) + # TODO: What is this for? - name: netcoreVersionTestUtils type: string @@ -109,6 +104,10 @@ parameters: - Release # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string @@ -138,11 +137,6 @@ parameters: - name: timeout type: number - # True if this is an ARM64 job. - - name: isArm64 - type: boolean - default: false - # True to use managed SNI, where applicable. - name: usemanagedSNI type: boolean @@ -153,7 +147,7 @@ parameters: type: string jobs: -- job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }} +- job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.poolImage, 'unknown_image')) }} # Some of our tests take longer than the default 60 minutes to run on some # OSes and configurations. @@ -161,11 +155,14 @@ jobs: pool: name: '${{ parameters.poolName }}' - ${{ if eq(parameters.hostedPool, true) }}: - vmImage: ${{ parameters.image }} + + # Images provided by Azure Pipelines must be selected using 'vmImage'. + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: + vmImage: ${{ parameters.poolImage }} + # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.image }} + - imageOverride -equals ${{ parameters.poolImage }} variables: - name: dotnetx86RootPath @@ -194,17 +191,20 @@ jobs: artifactName: ${{ parameters.mdsArtifactsName }} targetPath: $(Build.SourcesDirectory)/packages + - task: DownloadPipelineArtifact@2 + displayName: Download SqlServer Package Artifacts + inputs: + artifactName: ${{ parameters.sqlServerArtifactsName }} + targetPath: $(Build.SourcesDirectory)/packages + # Install the .NET SDK and Runtimes. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} - ${{ if parameters.isArm64 }}: - architecture: arm64 - # ARM64 needs to specify slightly different runtime version formats. - # See the install-dotnet template docs for more info. - runtimes: ['8.0', '9.0'] - ${{ else }}: - runtimes: [8.x, 9.x] + runtimes: [8.x, 9.x] + + # Restore dotnet CLI tools (e.g. pwsh, apicompat) before building. + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - ${{ if ne(parameters.prebuildSteps, '') }}: - ${{ parameters.prebuildSteps }} # extra steps to run before the build like downloading sni and the required configuration @@ -215,10 +215,9 @@ jobs: build: allNoDocs buildConfiguration: ${{ parameters.buildConfiguration }} referenceType: Project - assemblyBuildNumber: $(assemblyBuildNumber) - ${{ if ne(parameters.configProperties, '{}') }}: - - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml@self # update config.json file + - template: /eng/pipelines/common/templates/steps/update-config-file-step.yml@self # update config.jsonc file parameters: debug: ${{ parameters.debug }} saPassword: ${{ parameters.saPassword }} @@ -261,8 +260,6 @@ jobs: AliasName: ${{ parameters.configProperties.AliasName }} ${{ if parameters.configProperties.SupportsIntegratedSecurity }}: SupportsIntegratedSecurity: ${{ eq(parameters.configProperties.SupportsIntegratedSecurity, 'true') }} - ${{ if parameters.configProperties.SupportsFileStream }}: - SupportsFileStream: ${{ eq(parameters.configProperties.SupportsFileStream, 'true') }} ${{ if parameters.configProperties.DNSCachingConnString }}: DNSCachingConnString: ${{ parameters.configProperties.DNSCachingConnString }} ${{ if parameters.configProperties.DNSCachingServerCR }}: @@ -275,8 +272,6 @@ jobs: IsDNSCachingSupportedCR: ${{ eq(parameters.configProperties.IsDNSCachingSupportedCR, 'true') }} ${{ if parameters.configProperties.IsDNSCachingSupportedTR }}: IsDNSCachingSupportedTR: ${{ eq(parameters.configProperties.IsDNSCachingSupportedTR, 'true') }} - ${{ if parameters.configProperties.IsAzureSynapse }}: - IsAzureSynapse: ${{ eq(parameters.configProperties.IsAzureSynapse, 'true') }} ${{ if parameters.configProperties.ManagedIdentitySupported }}: ManagedIdentitySupported: ${{ eq(parameters.configProperties.ManagedIdentitySupported, 'true') }} @@ -326,6 +321,33 @@ jobs: ${{ if parameters.configProperties.FileStreamDirectory }}: fileStreamDirectory: ${{ parameters.configProperties.FileStreamDirectory }} + # Set up for x86 tests by manually installing dotnet for x86 to an alternative location. This + # is only used to execute the test runtime (framework to test is specified in build params), so + # it should be acceptable to just install a specific version in all cases. + # @TODO: This setup is very confusing. Ideally we should just be utilizing the dotnet installation + # earlier in the job. There has to be a cleaner way of doing this. + - ${{ if and(eq(parameters.enableX86Test, true), eq(parameters.operatingSystem, 'Windows')) }}: + - ${{ if ne(variables['dotnetx86RootPath'], '') }}: + # Install the .NET SDK and Runtimes for x86. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + forceX86: true + debug: ${{ parameters.debug }} + installDir: $(dotnetx86RootPath) + runtimes: [8.x, 9.x] + + # Ensure TestResults directory exists so that publish steps don't fail when + # tests are skipped due to a setup failure. + - pwsh: New-Item -ItemType Directory -Path TestResults -Force | Out-Null + displayName: 'Create TestResults Directory' + + # Gate: record that all setup steps (SDK install, build, SQL config, x86 SDK + # install, etc.) completed successfully. Test steps condition on this variable + # so they are skipped when setup fails, yet remain independent of each + # other's results. + - pwsh: Write-Host '##vso[task.setvariable variable=setupSucceeded]true' + displayName: 'Gate: Mark Setup Succeeded' + - ${{ if eq(parameters.enableX64Test, true) }}: # run native tests - template: /eng/pipelines/common/templates/steps/run-all-tests-step.yml@self # run tests parameters: @@ -335,25 +357,10 @@ jobs: referenceType: ${{ parameters.referenceType }} testSet: ${{ parameters.testSet }} operatingSystem: ${{ parameters.operatingSystem }} - abstractionsPackageVersion: ${{ parameters.abstractionsPackageVersion }} - loggingPackageVersion: ${{ parameters.loggingPackageVersion }} - mdsPackageVersion: ${{ parameters.mdsPackageVersion }} + packageVersion: ${{ parameters.packageVersion }} + sqlServerPackageVersion: ${{ parameters.sqlServerPackageVersion }} - ${{ if and(eq(parameters.enableX86Test, true), eq(parameters.operatingSystem, 'Windows')) }}: - # Set up for x86 tests by manually installing dotnet for x86 to an alternative location. This - # is only used to execute the test runtime (framework to test is specified in build params), so - # it should be acceptable to just install a specific version in all cases. - # @TODO: This setup is very confusing. Ideally we should just be utilizing the dotnet installation - # earlier in the job. There has to be a cleaner way of doing this. - - ${{ if ne(variables['dotnetx86RootPath'], '') }}: - # Install the .NET SDK and Runtimes for x86. - - template: /eng/pipelines/steps/install-dotnet.yml@self - parameters: - architecture: x86 - debug: ${{ parameters.debug }} - installDir: $(dotnetx86RootPath) - runtimes: [8.x, 9.x] - - template: /eng/pipelines/common/templates/steps/run-all-tests-step.yml@self parameters: debug: ${{ parameters.debug }} @@ -364,9 +371,8 @@ jobs: msbuildArchitecture: x86 dotnetx86RootPath: $(dotnetx86RootPath) operatingSystem: ${{ parameters.operatingSystem }} - abstractionsPackageVersion: ${{ parameters.abstractionsPackageVersion }} - loggingPackageVersion: ${{ parameters.loggingPackageVersion }} - mdsPackageVersion: ${{ parameters.mdsPackageVersion }} + packageVersion: ${{ parameters.packageVersion }} + sqlServerPackageVersion: ${{ parameters.sqlServerPackageVersion }} - template: /eng/pipelines/common/templates/steps/publish-test-results-step.yml@self parameters: diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml index fd87d7d9ac..29e0042569 100644 --- a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml +++ b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml @@ -11,10 +11,6 @@ parameters: - name: abstractionsArtifactsName type: string - # The version of the Abstractions package to depend on when referenceType is 'Package'. - - name: abstractionsPackageVersion - type: string - # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -37,18 +33,15 @@ parameters: type: string default: Logging.Artifacts - # The version of the Logging package to depend on when referenceType is 'Package'. - - name: loggingPackageVersion - type: string - # The name of the SqlClient pipeline artifacts to download. - name: mdsArtifactsName type: string default: MDS.Artifacts - # The version of the SqlClient package to depend on when referenceType is 'Package'. - - name: mdsPackageVersion + # The name of the SqlServer pipeline artifacts to download. + - name: sqlServerArtifactsName type: string + default: SqlServer.Artifacts # Jobs to run after the test jobs complete, if any. - name: postTestJobs @@ -84,6 +77,7 @@ stages: - stage: ${{ image.key }} dependsOn: - secrets_stage + - compute_versions_ci - ${{ each dep in parameters.additionalDependsOn }}: - ${{ dep }} @@ -92,6 +86,14 @@ stages: - name: saPassword value: $[stageDependencies.secrets_stage.secrets_job.outputs['SaPassword.Value']] + # Bring the computed versions into scope so the $(packageVersion) and + # $(sqlServerPackageVersion) macros passed to the test jobs resolve when + # building/restoring in Package mode. These mirror the build stages. + - name: packageVersion + value: $[ stageDependencies.compute_versions_ci.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions_ci.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + jobs: - ${{ each targetFramework in config.value.TargetFrameworks }}: - ${{ each platform in config.value.buildPlatforms }}: @@ -104,23 +106,21 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - hostedPool: ${{ eq(config.value.hostedPool, true) }} - image: ${{ image.value }} + poolImage: ${{ image.value }} jobDisplayName: ${{ format('{0}_{1}_{2}', replace(targetFramework, '.', '_'), platform, testSet) }} configProperties: ${{ config.value.configProperties }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} - abstractionsPackageVersion: ${{ parameters.abstractionsPackageVersion }} + packageVersion: $(packageVersion) loggingArtifactsName: ${{ parameters.loggingArtifactsName }} - loggingPackageVersion: ${{ parameters.loggingPackageVersion }} mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - mdsPackageVersion: ${{ parameters.mdsPackageVersion }} + sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} + sqlServerPackageVersion: $(sqlServerPackageVersion) prebuildSteps: ${{ parameters.prebuildSteps }} targetFramework: ${{ targetFramework }} netcoreVersionTestUtils: ${{config.value.netcoreVersionTestUtils }} testSet: ${{ testSet }} configSqlFor: ${{ config.value.configSqlFor }} operatingSystem: ${{ config.value.operatingSystem }} - isArm64: ${{ eq(config.value.isArm64, 'true') }} saPassword: $(saPassword) ${{if ne(config.value.configProperties, '{}') }}: ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}: @@ -136,8 +136,7 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - hostedPool: ${{ eq(config.value.hostedPool, true) }} - image: ${{ image.value }} + poolImage: ${{ image.value }} ${{if eq(usemanagedSNI, 'true') }}: jobDisplayName: ${{ format('{0}_{1}_{2}_{3}', replace(targetFramework, '.', '_'), platform, 'ManagedSNI', testSet) }} ${{ else }}: @@ -145,18 +144,17 @@ stages: configProperties: ${{ config.value.configProperties }} useManagedSNI: ${{ useManagedSNI }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} - abstractionsPackageVersion: ${{ parameters.abstractionsPackageVersion }} + packageVersion: $(packageVersion) loggingArtifactsName: ${{ parameters.loggingArtifactsName }} - loggingPackageVersion: ${{ parameters.loggingPackageVersion }} mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - mdsPackageVersion: ${{ parameters.mdsPackageVersion }} + sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} + sqlServerPackageVersion: $(sqlServerPackageVersion) prebuildSteps: ${{ parameters.prebuildSteps }} targetFramework: ${{ targetFramework }} netcoreVersionTestUtils: ${{config.value.netcoreVersionTestUtils }} testSet: ${{ testSet }} configSqlFor: ${{ config.value.configSqlFor }} operatingSystem: ${{ config.value.operatingSystem }} - isArm64: ${{ eq(config.value.isArm64, 'true') }} saPassword: $(saPassword) ${{if and(eq(usemanagedSNI, false), ne(config.value.configProperties, '{}')) }}: ${{ each x86TF in config.value.configProperties.x86TestTargetFrameworks }}: diff --git a/eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml b/eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml index 8c1d665e60..89ffb43e6c 100644 --- a/eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml +++ b/eng/pipelines/common/templates/steps/build-and-run-tests-netcore-step.yml @@ -4,13 +4,11 @@ # See the LICENSE file in the project root for more information. # ################################################################################# parameters: - - name: abstractionsPackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging). They + # all share this version. + - name: packageVersion type: string - default: $(abstractionsPackageVersion) - - - name: mdsPackageVersion - type: string - default: $(mdsPackageVersion) + default: $(packageVersion) - name: platform type: string @@ -48,8 +46,7 @@ steps: -p:TargetNetCoreVersion=${{ parameters.TargetNetCoreVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category!=failing&category!=flaky&category!=interactive" @@ -69,8 +66,7 @@ steps: -p:TargetNetCoreVersion=${{ parameters.TargetNetCoreVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category=flaky" @@ -91,8 +87,7 @@ steps: -p:TargetNetCoreVersion=${{ parameters.TargetNetCoreVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category!=failing&category!=flaky&category!=interactive" @@ -113,8 +108,7 @@ steps: -p:TargetNetCoreVersion=${{ parameters.TargetNetCoreVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category=flaky" diff --git a/eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml b/eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml index c13256c614..0fe2bd024a 100644 --- a/eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml +++ b/eng/pipelines/common/templates/steps/build-and-run-tests-netfx-step.yml @@ -4,13 +4,11 @@ # See the LICENSE file in the project root for more information. # ################################################################################# parameters: - - name: abstractionsPackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging). They + # all share this version. + - name: packageVersion type: string - default: $(abstractionsPackageVersion) - - - name: mdsPackageVersion - type: string - default: $(mdsPackageVersion) + default: $(packageVersion) - name: platform type: string @@ -48,8 +46,7 @@ steps: -p:TargetNetFxVersion=${{ parameters.TargetNetFxVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category!=failing&category!=flaky&category!=interactive" @@ -69,8 +66,7 @@ steps: -p:TargetNetFxVersion=${{ parameters.TargetNetFxVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category=flaky" @@ -91,8 +87,7 @@ steps: -p:TargetNetFxVersion=${{ parameters.TargetNetFxVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category!=failing&category!=flaky&category!=interactive" @@ -113,8 +108,7 @@ steps: -p:TargetNetFxVersion=${{ parameters.TargetNetFxVersion }} -p:ReferenceType=Package -p:Configuration=Release - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} --no-build -v n --filter "category=flaky" diff --git a/eng/pipelines/common/templates/steps/ci-project-build-step.yml b/eng/pipelines/common/templates/steps/ci-project-build-step.yml index e496aa2b71..c80de64fd4 100644 --- a/eng/pipelines/common/templates/steps/ci-project-build-step.yml +++ b/eng/pipelines/common/templates/steps/ci-project-build-step.yml @@ -46,149 +46,47 @@ parameters: - all - allNoDocs - # The build number suffix to apply to all assembly file versions built by this step. - - name: assemblyBuildNumber + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging, and the + # AKV Provider). They all share this version. Used when referenceType is Package; ignored for + # Project. + - name: packageVersion type: string + default: $(packageVersion) - # Necessary to build MDS when referenceType is Package. Ignored when referenceType is Project. - - name: abstractionsPackageVersion + # Necessary when referenceType is Package. Ignored when referenceType is Project. + - name: sqlServerPackageVersion type: string - default: $(abstractionsPackageVersion) - - # Necessary to build MDS when referenceType is Package. Ignored when referenceType is Project. - - name: loggingPackageVersion - type: string - default: $(loggingPackageVersion) - - # Necessary to build AKV Provider when referenceType is Package. Ignored when referenceType is Project. - - name: mdsPackageVersion - type: string - default: $(mdsPackageVersion) - - # Necessary to build AKV Provider when referenceType is Package. Ignored when referenceType is Project. - - name: akvPackageVersion - type: string - default: $(akvPackageVersion) + default: $(sqlServerPackageVersion) steps: -- ${{ if or(eq(parameters.operatingSystem, 'Windows'), eq(parameters.operatingSystem, 'deferedToRuntime')) }}: - - ${{ if or(eq(parameters.build, 'MDS'), eq(parameters.build, 'all'), eq(parameters.build, 'allNoDocs')) }}: - - task: MSBuild@1 - displayName: 'Restore [Win]' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - inputs: - solution: build.proj - msbuildArchitecture: x64 - msbuildArguments: >- - -t:restore - -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - retryCountOnTaskFailure: 1 - - - ${{ if eq(parameters.build, 'allNoDocs') }}: - - task: MSBuild@1 - displayName: 'Build Driver (no docs) [Win]' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - inputs: - solution: build.proj - msbuildArchitecture: x64 - platform: '${{ parameters.platform }}' - configuration: '${{ parameters.buildConfiguration }}' - msbuildArguments: >- - -t:BuildAllConfigurations - -p:ReferenceType=${{ parameters.referenceType }} - -p:GenerateNuget=false - -p:GenerateDocumentationFile=false - -p:BuildNumber=${{ parameters.buildNumber }} - -p:AssemblyBuildNumber=${{ parameters.assemblyBuildNumber }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - - - ${{ if or(eq(parameters.build, 'MDS'), eq(parameters.build, 'all')) }}: - - task: MSBuild@1 - displayName: 'Build Driver [Win]' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - inputs: - solution: build.proj - msbuildArchitecture: x64 - platform: '${{ parameters.platform }}' - configuration: '${{ parameters.buildConfiguration }}' - msbuildArguments: - -t:BuildAllConfigurations - -p:ReferenceType=${{ parameters.referenceType }} - -p:GenerateNuget=false - -p:BuildNumber=${{ parameters.buildNumber }} - -p:AssemblyBuildNumber=${{ parameters.assemblyBuildNumber }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - - - ${{ if or(eq(parameters.build, 'AkvProvider'), eq(parameters.build, 'all'), eq(parameters.build, 'allNoDocs')) }}: - - task: MSBuild@1 - displayName: 'Build AKV Provider' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - inputs: - solution: build.proj - msbuildArchitecture: x64 - platform: '${{ parameters.platform }}' - configuration: '${{ parameters.buildConfiguration }}' - msbuildArguments: >- - -t:BuildAkvProvider - -p:ReferenceType=${{ parameters.referenceType }} - -p:GenerateNuget=false - -p:BuildNumber=${{ parameters.buildNumber }} - -p:AssemblyBuildNumber=${{ parameters.assemblyBuildNumber }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:AkvPackageVersion=${{ parameters.akvPackageVersion }} - -- ${{ if or(eq(parameters.operatingSystem, 'Linux'), eq(parameters.operatingSystem, 'MacOS'), eq(parameters.operatingSystem, 'deferedToRuntime')) }}: + # Build MDS - ${{ if or(eq(parameters.build, 'MDS'), eq(parameters.build, 'all'), eq(parameters.build, 'allNoDocs')) }}: - task: DotNetCoreCLI@2 - displayName: 'Build SqlClient [${{ parameters.operatingSystem }}]' - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) + displayName: 'Build Driver' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) inputs: - command: custom + command: build projects: build.proj - custom: build arguments: >- -t:BuildSqlClient - -p:ReferenceType=${{ parameters.referenceType }} - -p:TestEnabled=true - -p:GenerateNuget=false - -p:GenerateDocumentationFile=false -p:Configuration=${{ parameters.buildConfiguration }} + -p:ReferenceType=${{ parameters.referenceType }} -p:BuildNumber=${{ parameters.buildNumber }} - -p:AssemblyBuildNumber=${{ parameters.assemblyBuildNumber }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - verbosityRestore: Detailed - verbosityPack: Detailed + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + # Build AKV Provider - ${{ if or(eq(parameters.build, 'AkvProvider'), eq(parameters.build, 'all'), eq(parameters.build, 'allNoDocs')) }}: - task: DotNetCoreCLI@2 - displayName: 'Build AKV Provider [${{ parameters.operatingSystem }}]' - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) + displayName: 'Build AKV Provider' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) inputs: - command: custom + command: build projects: build.proj - custom: build arguments: >- -t:BuildAkvProvider - -p:ReferenceType=${{ parameters.referenceType }} - -p:TestEnabled=true - -p:GenerateNuget=false - -p:GenerateDocumentationFile=false -p:Configuration=${{ parameters.buildConfiguration }} + -p:ReferenceType=${{ parameters.referenceType }} -p:BuildNumber=${{ parameters.buildNumber }} - -p:AssemblyBuildNumber=${{ parameters.assemblyBuildNumber }} - -p:AkvPackageVersion=${{ parameters.akvPackageVersion }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - verbosityRestore: Detailed - verbosityPack: Detailed + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml index 15de459d50..bafb9eda1c 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml @@ -4,8 +4,8 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Linux host. For example, our 1ES -# Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and +# This step configures an existing SQL Server running on the local Linux host. For example, our +# 1ES pools have images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: @@ -16,8 +16,16 @@ parameters: steps: + # Make sure sqlcmd is available; the SQL Server images do not always ship it. + - template: /eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml@self + # Configure SQL Server. - bash: | + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." + exit 1 + fi + sudo systemctl stop mssql-server # Password for the SA user (required) @@ -39,10 +47,11 @@ steps: do echo Waiting for SQL Server to start... sleep 3s - /opt/mssql-tools/bin/sqlcmd \ + "$SQLCMD_BIN" \ -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "SELECT @@VERSION" 2>/dev/null errstatus=$? ((counter++)) @@ -55,3 +64,6 @@ steps: exit $errstatus fi displayName: 'Configure SQL Server [Linux]' + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml index c1b788717d..0ffeb4f2ab 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml @@ -29,90 +29,270 @@ steps: export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] ' set -x - # Install Docker and SQLCMD tools. + # Install Colima (which provides the Docker daemon, since Docker Desktop is + # not available here), the docker CLI, and sqlcmd. brew install colima - brew install --cask docker - brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release - brew update - HOMEBREW_ACCEPT_EULA=Y brew install mssql-tools18 - colima start --arch x86_64 + + # Homebrew ships no Intel macOS bottle for the current docker formula, so + # 'brew install docker' compiles the CLI (and builds Go to do it), which + # takes minutes and often exhausts the step timeout. Install the newest + # bottled version directly instead. + DOCKER_CLI_DIR="$HOME/.docker-cli/bin" + if ! pwsh -NoProfile -File "$(Build.SourcesDirectory)/eng/pipelines/scripts/Install-DockerCli.macos.ps1" -DestinationPath "$DOCKER_CLI_DIR"; then + echo "ERROR: Failed to install the docker CLI." + exit 1 + fi + # prependpath only affects later steps, so fix PATH for this one as well. + export PATH="$DOCKER_CLI_DIR:$PATH" + + # go-sqlcmd, rather than mssql-tools18 from the microsoft/mssql-release + # tap. That tap pins an openssl@3 formula that has no Intel bottle, so + # installing it compiled openssl from source and cost 12.5 minutes of the + # step budget. go-sqlcmd is a single bottled Go binary with no openssl + # dependency, and accepts the same flags used below. + brew install sqlcmd + + # Fail fast if sqlcmd was not installed. Without this check the script + # would loop for ~6 minutes trying to connect. + if ! command -v sqlcmd &>/dev/null; then + echo "ERROR: sqlcmd is not on PATH after 'brew install sqlcmd'." + exit 1 + fi + + # Start Colima with Virtualization.framework for x86_64 binary translation + # on Apple Silicon. Rosetta/binfmt emulation is enabled by default in + # Colima >= 0.8 when using --vm-type vz, which is dramatically faster than + # --arch x86_64 (full QEMU VM emulation). + # Requires macOS >= 13 (Ventura). + # + # The Lima hostagent that backs Colima intermittently fails to become + # ready within its short startup window on the macOS CI runners + # ("hostagent did not start up in 5s"), which is transient. Retry a few + # times, tearing down the half-created instance between attempts and + # dumping the hostagent log for diagnostics, before giving up. + colimaAttempts=3 + colimaStarted=0 + for ((c=1; c<=colimaAttempts; c++)) + do + echo "Starting Colima (attempt #$c of $colimaAttempts)..." + if colima start --vm-type vz --cpu 4 --memory 4; then + colimaStarted=1 + break + fi + echo "colima start failed (attempt #$c of $colimaAttempts)." + echo "--- hostagent stderr (last 40 lines) ---" + tail -40 "$HOME/.colima/_lima/colima/ha.stderr.log" 2>/dev/null || true + echo "--- Tearing down half-created instance before retry ---" + colima delete --force || true + sleep 5 + done + + if [ $colimaStarted -ne 1 ]; then + echo "ERROR: colima failed to start after $colimaAttempts attempts." + colima status || true + exit 1 + fi + + # Point the docker CLI at Colima's daemon socket. Colima normally sets an + # active docker context, but the standalone docker CLI installed above can + # default to unix:///var/run/docker.sock, which + # does not exist on macOS without Docker Desktop. This caused every + # 'docker pull' to fail instantly with: + # failed to connect to the docker API at unix:///var/run/docker.sock + # Export DOCKER_HOST explicitly so all docker commands reach Colima. + export DOCKER_HOST="unix://${HOME}/.colima/default/docker.sock" docker --version - docker pull mcr.microsoft.com/mssql/server:2025-latest + + # Verify the docker daemon is actually reachable before attempting to pull. + # This fails fast with useful diagnostics instead of looping through the + # pull retries when the socket is misconfigured. + if ! docker info >/dev/null 2>&1; then + echo "ERROR: docker daemon is not reachable at $DOCKER_HOST." + echo "--- colima status ---" + colima status || true + echo "--- docker context ls ---" + docker context ls || true + exit 1 + fi + + # Pull the SQL Server image with retries. The macOS agents intermittently + # fail to resolve or reach mcr.microsoft.com (DNS lookups or registry i/o + # time out), which previously failed the whole step on the first attempt. + # Retry a handful of times with a fixed delay before giving up. + pullAttempts=5 + pullDelay=15 + pulled=0 + for ((p=1; p<=pullAttempts; p++)) + do + echo "Pulling SQL Server image (attempt #$p of $pullAttempts)..." + if docker pull --platform linux/amd64 mcr.microsoft.com/mssql/server:2025-latest; then + pulled=1 + break + fi + echo "docker pull failed (attempt #$p of $pullAttempts). Retrying in ${pullDelay}s..." + sleep $pullDelay + done + + if [ $pulled -ne 1 ]; then + echo "ERROR: Unable to pull mcr.microsoft.com/mssql/server:2025-latest after $pullAttempts attempts (registry/DNS unreachable)." + exit 1 + fi # Password for the SA user (required) MSSQL_SA_PW="${{ parameters.saPassword }}" - docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2025-latest + # Collect everything needed to diagnose a container that crashed or never + # became ready. + # + # The mssql dump collector (paldumper) writes hundreds of + # "find: '/proc/N/...': Permission denied" lines to the container's stderr + # whenever sqlservr faults. That noise used to fill the entire captured + # window and hide the actual SQL Server error, so it is filtered out here + # and both the head and the tail of the log are printed. + dumpSqlDiagnostics() + { + set +x - sleep 5 + echo "--- Container status ---" + docker ps -a --filter "name=^/sql1$" || true - docker ps -a + echo "--- Container state ---" + docker inspect sql1 --format 'ExitCode={{.State.ExitCode}} OOMKilled={{.State.OOMKilled}} Error="{{.State.Error}}" StartedAt={{.State.StartedAt}} FinishedAt={{.State.FinishedAt}}' || true - # Connect to the SQL Server container and get its version. - # - # It can take a while for the docker container to start listening and be - # ready for connections, so we will wait for up to 2 minutes, checking every - # 3 seconds. + echo "--- SQL Server errorlog (last 80 lines) ---" + # docker cp works against a stopped container, so this survives a crash. + if docker cp sql1:/var/opt/mssql/log/errorlog "$SQL_ERRORLOG" 2>/dev/null; then + tail -80 "$SQL_ERRORLOG" + else + echo "(errorlog not available)" + fi + + echo "--- Container logs, dump-collector noise filtered (first 60 lines) ---" + docker logs sql1 2>&1 | grep -vE "^(find|dmesg|timeout): " | head -60 || true + echo "--- Container logs, dump-collector noise filtered (last 40 lines) ---" + docker logs sql1 2>&1 | grep -vE "^(find|dmesg|timeout): " | tail -40 || true + + echo "--- sqlcmd errors ---" + cat "$SQLCMD_ERRORS" 2>/dev/null || echo "(none)" - # Wait 3 seconds between attempts. - delay=3 + echo "--- Host capacity ---" + echo "hw.ncpu=$(sysctl -n hw.ncpu 2>/dev/null) hw.memsize=$(sysctl -n hw.memsize 2>/dev/null)" + vm_stat || true - # Try up to 40 times (2 minutes) to connect. - maxAttempts=40 + echo "--- Guest capacity ---" + colima ssh -- free -m || true - # Attempt counter. - attempt=1 + set -x + } - # Flag to indicate when SQL Server is ready to accept connections. - ready=0 + SQL_ERRORLOG=$(Agent.TempDirectory)/mssql_errorlog - while [ $attempt -le $maxAttempts ] + # Start SQL Server, retrying the whole container lifecycle. sqlservr + # intermittently core dumps within a minute or two of starting on these + # agents (the container goes to "Exited (1)" while the readiness loop is + # still polling), which previously failed the entire step on the first + # occurrence. Colima startup and the image pull already retry; the + # container did not. + runAttempts=3 + sqlReady=0 + + for ((r=1; r<=runAttempts; r++)) do + echo "Starting SQL Server container (attempt #$r of $runAttempts)..." + + # Clear any container and sqlcmd output left behind by a prior attempt. + docker rm -f sql1 >/dev/null 2>&1 || true + : > "$SQLCMD_ERRORS" + + if ! docker run --platform linux/amd64 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2025-latest; then + echo "ERROR: Failed to start the sql1 container (attempt #$r of $runAttempts)." + dumpSqlDiagnostics + continue + fi + + sleep 10 + + docker ps -a - echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + # Connect to the SQL Server container and get its version. + # + # With Rosetta 2 emulation, SQL Server starts much faster than under full + # QEMU emulation, but it can still take a minute or two. We allow up to + # 6 minutes (72 attempts × 5 seconds) as a generous upper bound. - sqlcmd -S 127.0.0.1 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1 + # Wait 5 seconds between attempts. + delay=5 - # If the command was successful, then the SQL Server is ready. - if [ $? -eq 0 ]; then - ready=1 + # Try up to 72 times (~6 minutes) to connect. + maxAttempts=72 + + # Attempt counter. + attempt=1 + + # Set when the container died before SQL Server accepted connections. + crashed=0 + + while [ $attempt -le $maxAttempts ] + do + + echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + + # -C trusts the self-signed certificate inside the container. + if sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> "$SQLCMD_ERRORS" 2>&1; then + sqlReady=1 + break + fi + + # Verify the container is still running; no point retrying if it crashed. + if ! docker ps --filter "name=^/sql1$" --filter "status=running" --format '{{.Names}}' | grep -Fxq 'sql1'; then + echo "ERROR: sql1 container is no longer running (attempt #$r of $runAttempts)." + crashed=1 + break + fi + + # Increment the attempt counter. + ((attempt++)) + + # Wait before trying again. + sleep $delay + + done + + if [ $sqlReady -eq 1 ]; then break fi - # Increment the attempt counter. - ((attempt++)) - - # Wait before trying again. - sleep $delay + if [ $crashed -ne 1 ]; then + echo "ERROR: Cannot connect to SQL Server after $maxAttempts attempts (attempt #$r of $runAttempts)." + fi + dumpSqlDiagnostics done # Is the SQL Server ready? - if [ $ready -eq 0 ] + if [ $sqlReady -ne 1 ] then - # No, so report the error(s) and exit. - echo Cannot connect to SQL Server; installation aborted; errors were: - cat $SQLCMD_ERRORS - rm -f $SQLCMD_ERRORS + echo "ERROR: SQL Server did not become ready after $runAttempts container attempts; installation aborted." + rm -f "$SQLCMD_ERRORS" "$SQL_ERRORLOG" exit 1 fi - rm -f $SQLCMD_ERRORS + rm -f "$SQLCMD_ERRORS" "$SQL_ERRORLOG" echo "Use sqlcmd to show which IP addresses are being listened on..." echo 0.0.0.0 - sqlcmd -S 0.0.0.0 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + sqlcmd -S 0.0.0.0 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 echo 127.0.0.1 - sqlcmd -S 127.0.0.1 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 echo ::1 - sqlcmd -S ::1 -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + sqlcmd -S ::1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 echo localhost - sqlcmd -S localhost -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + sqlcmd -S localhost -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 echo "(sqlcmd default / not specified)" - sqlcmd -No -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 + sqlcmd -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2 echo "Configuring Dedicated Administer Connections to allow remote connections..." - sqlcmd -S 127.0.0.1 -No -U sa -P "$MSSQL_SA_PW" -Q "sp_configure 'remote admin connections', 1; RECONFIGURE;" + sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "sp_configure 'remote admin connections', 1; RECONFIGURE;" if [ $? = 1 ] then echo "Error configuring DAC for remote access." @@ -122,3 +302,10 @@ steps: fi displayName: 'Configure SQL Server [macOS]' + # Well above a healthy run, but low enough that a wedged install or Colima + # boot fails here instead of consuming the whole test job. Measured worst + # case is ~24 minutes: Colima boot ~6, the SQL image pull ~10 (7 of which is + # extraction inside the VM), and up to 6 more waiting for SQL to accept + # connections. The container is now started up to 3 times, so the readiness + # wait can cost 18 minutes rather than 6 in the pathological case. + timeoutInMinutes: 55 diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml index 749c30a79a..cd691cc4a2 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and +# 1ES pools have images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: @@ -62,8 +62,21 @@ parameters: type: string default: $(LocalDbSharedInstanceName) +# Step execution order: +# 1. Enable TCP, NP & Firewall — Enable TCP and Named Pipes protocols, open firewall ports +# 2. Create SQL user — Create test login/user, grant sysadmin, enable SA; restarts SQL if needed +# 3. Enable FileStream [Win] — (conditional) Enable FileStream via WMI; sp_configure deferred to step 8 +# 4. Create FileStreamFolder — (conditional) Create the FileStream data directory +# 5. Setup SQL Alias — Register TCP aliases in x86/x64 registry paths +# 6. Add SQL Certificate — Generate self-signed cert, trust it, bind to SQL Server, grant key access +# 7. Restart SQL Server — Restart to pick up protocol, cert, and FileStream WMI changes; wait for readiness +# 8. Configure FileStream Access — (conditional) Run sp_configure filestream_access_level after restart +# 9. Start SQL Server Browser — Start the Browser service for named-instance discovery +# 10. Enable LocalDB — (conditional) Start and share the LocalDB instance + steps: + # Step 1: Enable TCP, Named Pipes protocols and configure Windows Firewall rules. # GOTCHA: We must use the Windows-only powershell task here instead of the cross-platform pwsh # task because we call some Windows-specific cmdlets. - powershell: | @@ -108,6 +121,9 @@ steps: displayName: 'Enable TCP, NP & Firewall [Win]' retryCountOnTaskFailure: 2 + # Step 2: Create test login and user with sysadmin privileges, enable and set the SA password. + # If SQL Server is unresponsive (e.g. after protocol changes), this step will restart it once + # and retry before failing. - powershell: | $password = "${{ parameters.saPassword }}" @@ -119,11 +135,19 @@ steps: Write-Host $machineName Import-Module "sqlps" + + # Determine the Windows service name for this SQL instance. + $serviceName = "${{parameters.instanceName }}" + if ("${{parameters.instanceName }}" -ne "MSSQLSERVER") { + $serviceName = "MSSQL`$${{parameters.instanceName }}" + } + + $hasRestarted = $false $tries = 0 while ($true) { $tries++ try { - Invoke-Sqlcmd -ServerInstance "$machineName" @" + Invoke-Sqlcmd -ServerInstance "$machineName" -ConnectionTimeout 5 -ErrorAction Stop @" CREATE LOGIN [${{parameters.user }}] WITH PASSWORD=N'$password', DEFAULT_DATABASE=[master], DEFAULT_LANGUAGE=[us_english], CHECK_EXPIRATION=OFF, CHECK_POLICY=OFF; CREATE USER [${{parameters.user }}] FROM LOGIN [${{parameters.user }}]; @@ -133,11 +157,20 @@ steps: "@ break } catch { - if ($tries -ge 5) { - Write-Host "##[error]Failed to create database user after $tries tries." - break + if (-not $hasRestarted -and $tries -ge 5) { + # SQL Server may need a restart after protocol changes (TCP/NP). + Write-Host "Connection failed after $tries attempts. Restarting SQL Server ($serviceName) and retrying..." + Restart-Service -Name $serviceName -Force -ErrorAction Stop + $hasRestarted = $true + $tries = 0 + Start-Sleep -Seconds 5 + continue } - Write-Host "Failed to connect to server. Retrying in 5 seconds..." + if ($tries -ge 10) { + Write-Host "##[error]Failed to create database user after $tries tries (including a restart)." + throw + } + Write-Host "Failed to connect to server (attempt $tries). Retrying in 5 seconds..." Start-Sleep -Seconds 5 } } @@ -146,31 +179,21 @@ steps: SQL_USER: ${{parameters.user }} SQL_PASSWD: ${{ parameters.saPassword }} + # Step 3: Enable FileStream at the OS/WMI level (conditional on SQLRootPath being set). + # Only the WMI flag is toggled here. The T-SQL sp_configure call happens in step 8, + # after the full SQL Server restart in step 7, to avoid needing a double restart. - ${{ if ne(parameters.SQLRootPath, '') }}: - powershell: | - #Enable FileStream + #Enable FileStream via WMI. + # The sp_configure call is deferred to after the "Restart SQL Server [Win]" + # step so we don't need a separate restart here. $instance = "${{parameters.instanceName }}" $wmi = Get-WmiObject -Namespace "${{parameters.SQLRootPath }}" -Class FilestreamSettings | where {$_.InstanceName -eq $instance} $wmi.EnableFilestream(3, $instance) - - $machineName = $env:COMPUTERNAME - - if ("${{parameters.instanceName }}" -ne "MSSQLSERVER"){ - $machineName += "\${{parameters.instanceName }}" - } - - #Change the access level for FileStream for SQLServer - Set-ExecutionPolicy Unrestricted - Import-Module "sqlps" - Invoke-Sqlcmd -ServerInstance "$machineName" @" - EXEC sp_configure filestream_access_level, 2; - RECONFIGURE; - "@ + Write-Host "FileStream enabled via WMI for instance '$instance'." displayName: 'Enable FileStream [Win]' - env: - SQL_USER: ${{parameters.user }} - SQL_PASSWD: ${{ parameters.saPassword }} + # Step 4: Create the FileStream data directory (conditional on fileStreamDirectory being set). - ${{ if ne(parameters.FileStreamDirectory, '') }}: - powershell: | New-Item -Path ${{ parameters.fileStreamDirectory }} -ItemType Directory @@ -178,6 +201,8 @@ steps: retryCountOnTaskFailure: 1 continueOnError: true + # Step 5: Register TCP-based SQL Server aliases in both x86 and x64 registry hives + # so test connections using the alias name resolve to the correct host and port. - powershell: | $SQLServerName = ("{0}" -f [System.Net.Dns]::GetHostByName($env:computerName).HostName) Write-Host FQDN is: $SQLServerName @@ -196,6 +221,9 @@ steps: New-ItemProperty -Path ${{parameters.x64AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName displayName: 'Setup SQL Alias [Win]' + # Step 6: Generate a self-signed TLS certificate, add it to the trusted root store, + # bind it to all SQL Server instances, and grant the SQL service account read access + # to the private key. - powershell: | # Create Certificate $computerDnsName = [System.Net.Dns]::Resolve($null).HostName @@ -233,6 +261,9 @@ steps: } displayName: 'Add SQL Certificate [Win]' + # Step 7: Restart SQL Server to apply all preceding configuration changes (protocols, + # certificate, FileStream WMI). Waits for SQL Server to accept connections before + # proceeding (up to ~160s). - powershell: | # You need to restart SQL Server for the change to persist # -Force takes care of any dependent services, like SQL Agent. @@ -250,8 +281,69 @@ steps: Restart-Service -Name "$serviceName" -Force Restart-Service -Name MSSQLSERVER* -Force + # Wait for SQL Server to be ready to accept connections after restart. + # Worst-case budget: 20 attempts x (5s connection timeout + 3s sleep) = ~160s + $machineName = $env:COMPUTERNAME + if ("${{parameters.instanceName }}" -ne "MSSQLSERVER") { + $machineName += "\${{parameters.instanceName }}" + } + + Import-Module "sqlps" + $tries = 0 + while ($true) { + $tries++ + try { + Invoke-Sqlcmd -ServerInstance "$machineName" -Query "SELECT @@VERSION" -ConnectionTimeout 5 -ErrorAction Stop + Write-Host "SQL Server is ready after restart (attempt $tries)." + break + } catch { + if ($tries -ge 20) { + Write-Host "##[error]SQL Server did not become ready after $tries attempts." + throw + } + Write-Host "Waiting for SQL Server to start (attempt $tries/20)..." + Start-Sleep -Seconds 3 + } + } + displayName: 'Restart SQL Server [Win]' + # Step 8: Configure FileStream access level via T-SQL after the restart so we don't + # need a separate restart in the "Enable FileStream [Win]" step. + - ${{ if ne(parameters.SQLRootPath, '') }}: + - powershell: | + $machineName = $env:COMPUTERNAME + if ("${{parameters.instanceName }}" -ne "MSSQLSERVER") { + $machineName += "\${{parameters.instanceName }}" + } + + Set-ExecutionPolicy Unrestricted + Import-Module "sqlps" + + # Immediately after the restart SQL Server can still be stabilizing, which + # can surface transient errors such as "the session is in the kill state" + # or other severe command errors. Retry a few times before failing so a + # momentary hiccup does not fail the whole step. + $tries = 0 + while ($true) { + $tries++ + try { + Invoke-Sqlcmd -ServerInstance "$machineName" -ConnectionTimeout 5 -Query "EXEC sp_configure filestream_access_level, 2; RECONFIGURE;" -ErrorAction Stop + Write-Host "FileStream access level configured successfully (attempt $tries)." + break + } catch { + if ($tries -ge 10) { + Write-Host "##[error]Failed to configure FileStream access level after $tries attempts." + throw + } + Write-Host "FileStream configuration failed (attempt $tries/10): $($_.Exception.Message). Retrying..." + Start-Sleep -Seconds 5 + } + } + displayName: 'Configure FileStream Access Level [Win]' + + # Step 9: Start the SQL Server Browser service so that named instances can be + # discovered by clients. - powershell: | $arrService = Get-Service -Name "SQLBrowser" $arrService @@ -271,6 +363,7 @@ steps: } displayName: 'Start Sql Server Browser [Win]' + # Step 10: Start and share the LocalDB instance (conditional on enableLocalDB). - ${{ if parameters.enableLocalDB }}: - powershell: | #script to enable local db diff --git a/eng/pipelines/common/templates/steps/generate-nuget-package-step.yml b/eng/pipelines/common/templates/steps/generate-nuget-package-step.yml deleted file mode 100644 index 46c5c61492..0000000000 --- a/eng/pipelines/common/templates/steps/generate-nuget-package-step.yml +++ /dev/null @@ -1,80 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# -parameters: - - name: nuspecPath - type: string - - - name: packageVersion - type: string - - - name: outputDirectory - type: string - default: '$(Build.SourcesDirectory)/output' - - # The C# build configuration (e.g. Debug or Release) to use when building the NuGet package. - - name: buildConfiguration - type: string - default: Debug - values: - - Debug - - Release - - - name: generateSymbolsPackage - type: boolean - - - name: displayName - type: string - - - name: installNuget - type: boolean - default: true - - # The C# project reference type to use when building and packing the packages. - - name: referenceType - type: string - values: - # Reference sibling packages as NuGet packages. - - Package - # Reference sibling packages as C# projects. - - Project - - # Semi-colon separated properties to pass to nuget pack via the -properties argument. - - name: properties - type: string - default: '' - -steps: -- ${{ if parameters.installNuget }}: - - task: NuGetToolInstaller@1 - displayName: 'Install Latest Nuget' - inputs: - checkLatest: true - -- powershell: | - $Commit=git rev-parse HEAD - Write-Host "##vso[task.setvariable variable=CommitHead;]$Commit" - displayName: CommitHead - -- task: NuGetCommand@2 - displayName: ${{parameters.displayName }} - inputs: - command: custom - ${{ if parameters.generateSymbolsPackage }}: - arguments: >- - pack - -Symbols - -SymbolPackageFormat snupkg - ${{parameters.nuspecPath}} - -Version ${{parameters.packageVersion}} - -OutputDirectory ${{parameters.outputDirectory}} - -properties "COMMITID=$(CommitHead);Configuration=${{parameters.buildConfiguration}};ReferenceType=${{parameters.referenceType}};${{parameters.properties}}" - ${{else }}: - arguments: >- - pack - ${{parameters.nuspecPath}} - -Version ${{parameters.packageVersion}} - -OutputDirectory ${{parameters.outputDirectory}} - -properties "COMMITID=$(CommitHead);Configuration=${{parameters.buildConfiguration}};ReferenceType=${{parameters.referenceType}};${{parameters.properties}}" diff --git a/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml b/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml new file mode 100644 index 0000000000..d5e82f434b --- /dev/null +++ b/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml @@ -0,0 +1,83 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Ensures the SQL Server command line tools are available on a Linux agent. +# +# The SQL Server images in our 1ES pool (for example ADO-UB24-SQL25) ship the SQL Server engine but +# do not always ship the command line tools, so install them when they are missing. +# +# Sets two variables for the remaining steps in the job: +# +# SqlCmdBin - absolute path to the sqlcmd executable. +# SqlCmdTrustArg - '-C' when sqlcmd needs to be told to trust the server's self-signed +# certificate, otherwise empty. + +steps: + + - bash: | + set -u + + if ! command -v sudo >/dev/null 2>&1; then + echo "ERROR: 'sudo' is required to install the SQL Server command line tools." + exit 1 + fi + + find_sqlcmd() { + if command -v sqlcmd >/dev/null 2>&1; then + command -v sqlcmd + return 0 + fi + # mssql-tools default install locations. + local candidate + for candidate in /opt/mssql-tools18/bin/sqlcmd /opt/mssql-tools/bin/sqlcmd; do + if [ -x "$candidate" ]; then + echo "$candidate" + return 0 + fi + done + return 1 + } + + if SQLCMD_BIN="$(find_sqlcmd)"; then + echo "Found existing sqlcmd at '$SQLCMD_BIN'." + else + echo "sqlcmd was not found; installing mssql-tools18..." + + # The Microsoft package repository is usually already configured on these images (that is + # where the engine came from), so try a plain install first and only register the repository + # if that fails. + sudo apt-get update + if ! sudo env ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev; then + # shellcheck disable=SC1091 + . /etc/os-release + echo "Registering the Microsoft package repository for Ubuntu $VERSION_ID..." + curl -sSL -o /tmp/packages-microsoft-prod.deb \ + "https://packages.microsoft.com/config/ubuntu/$VERSION_ID/packages-microsoft-prod.deb" + sudo dpkg -i /tmp/packages-microsoft-prod.deb + sudo apt-get update + sudo env ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev + fi + + if ! SQLCMD_BIN="$(find_sqlcmd)"; then + echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations," + echo " and installing mssql-tools18 did not provide it." + exit 1 + fi + fi + + # sqlcmd from mssql-tools18 (and go-sqlcmd) encrypts by default, so it needs -C to trust the + # local server's self-signed certificate. The older mssql-tools build does not support -C. + case "$SQLCMD_BIN" in + */mssql-tools/bin/sqlcmd) SQLCMD_TRUST_ARG="" ;; + *) SQLCMD_TRUST_ARG="-C" ;; + esac + + echo "Using sqlcmd '$SQLCMD_BIN' with trust argument '$SQLCMD_TRUST_ARG'." + + # Publish for the remaining steps in this job. + echo "##vso[task.setvariable variable=SqlCmdBin]$SQLCMD_BIN" + echo "##vso[task.setvariable variable=SqlCmdTrustArg]$SQLCMD_TRUST_ARG" + displayName: 'Install sqlcmd [Linux]' diff --git a/eng/pipelines/common/templates/steps/override-sni-version.yml b/eng/pipelines/common/templates/steps/override-sni-version.yml index 3b275262c3..b3496b9f1f 100644 --- a/eng/pipelines/common/templates/steps/override-sni-version.yml +++ b/eng/pipelines/common/templates/steps/override-sni-version.yml @@ -42,6 +42,19 @@ steps: # add the new package source $packageSources.AppendChild($newSource) + # Exact mappings take precedence over the governed feed's wildcard, ensuring validation SNI + # packages are restored from this source. Both package IDs are externally produced and are + # therefore intentionally not eligible for the repository's local feed. + $packageSourceMapping = $xml.SelectSingleNode('//ns:packageSourceMapping', $nsm) + $newMapping = $xml.CreateElement("packageSource") + $newMapping.SetAttribute("key","SNIValidation") + foreach ($packageId in @("Microsoft.Data.SqlClient.SNI", "Microsoft.Data.SqlClient.SNI.runtime")) { + $package = $xml.CreateElement("package") + $package.SetAttribute("pattern", $packageId) + $newMapping.AppendChild($package) + } + $packageSourceMapping.AppendChild($newMapping) + # save the xml file $xml.Save($NugetCfg) type $NugetCfg diff --git a/eng/pipelines/common/templates/steps/pre-build-step.yml b/eng/pipelines/common/templates/steps/pre-build-step.yml index 068223af70..1d57d18296 100644 --- a/eng/pipelines/common/templates/steps/pre-build-step.yml +++ b/eng/pipelines/common/templates/steps/pre-build-step.yml @@ -5,7 +5,7 @@ ################################################################################# steps: # Install the .NET SDK and Runtimes. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: runtimes: [8.x, 9.x] diff --git a/eng/pipelines/common/templates/steps/publish-test-results-step.yml b/eng/pipelines/common/templates/steps/publish-test-results-step.yml index 9cb356aa45..8e06e9dece 100644 --- a/eng/pipelines/common/templates/steps/publish-test-results-step.yml +++ b/eng/pipelines/common/templates/steps/publish-test-results-step.yml @@ -59,9 +59,22 @@ steps: Get-ChildItem -Filter "*.coverage" -Recurse displayName: '[Debug] List test result coverage files' +# When an earlier step fails (e.g. SQL Server setup), no TestResults directory is +# produced and PublishPipelineArtifact fails with "Path does not exist", adding a +# second error that buries the real one. Gate the publish on the directory +# actually existing. +- pwsh: | + $exists = Test-Path -Path 'TestResults' -PathType Container + if (-not $exists) { + Write-Host 'No TestResults directory was produced; skipping test artifact publish.' + } + Write-Host "##vso[task.setvariable variable=HasTestResults]$($exists.ToString().ToLowerInvariant())" + displayName: 'Check for test results' + condition: succeededOrFailed() + - task: PublishPipelineArtifact@1 displayName: 'Publish Test Artifacts' inputs: targetPath: TestResults artifact: '${{parameters.targetFramework }}WinAz$(System.JobId)' - condition: succeededOrFailed() + condition: and(succeededOrFailed(), eq(variables['HasTestResults'], 'true')) diff --git a/eng/pipelines/common/templates/steps/run-all-tests-step.yml b/eng/pipelines/common/templates/steps/run-all-tests-step.yml index b1d0261abc..928c2e3abc 100644 --- a/eng/pipelines/common/templates/steps/run-all-tests-step.yml +++ b/eng/pipelines/common/templates/steps/run-all-tests-step.yml @@ -5,10 +5,9 @@ ################################################################################# parameters: - - name: abstractionsPackageVersion - type: string - - - name: loggingPackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging). They + # all share this version. + - name: packageVersion type: string # True to emit debug information and steps. @@ -19,7 +18,7 @@ parameters: - name: targetFramework type: string - - name: mdsPackageVersion + - name: sqlServerPackageVersion type: string - name: platform @@ -76,303 +75,280 @@ steps: condition: succeededOrFailed() - ${{if eq(parameters.operatingSystem, 'Windows')}}: - - ${{if eq(parameters.referenceType, 'Project')}}: - - task: MSBuild@1 - displayName: 'Run Unit Tests ${{parameters.msbuildArchitecture }}' - condition: succeededOrFailed() - inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunUnitTests - -p:TF=${{ parameters.targetFramework }} - -p:ReferenceType=${{ parameters.referenceType }} - ${{ else }}: # x86 - msbuildArguments: >- - -t:RunUnitTests - -p:TF=${{ parameters.targetFramework }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + - task: DotNetCoreCLI@2 + displayName: 'Run Unit Tests ${{parameters.msbuildArchitecture }}' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) + inputs: + command: build + projects: build.proj + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + arguments: >- + -t:TestSqlClientUnit + -p:TestFramework=${{ parameters.targetFramework }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestResultsFolderPath=TestResults + ${{ else }}: # x86 + arguments: >- + -t:TestSqlClientUnit + -p:TestFramework=${{ parameters.targetFramework }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + -p:TestResultsFolderPath=TestResults - - task: MSBuild@1 - displayName: 'Run Flaky Unit Tests ${{parameters.msbuildArchitecture }}' - condition: succeededOrFailed() - inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunUnitTests - -p:TF=${{ parameters.targetFramework }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - ${{ else }}: # x86 - msbuildArguments: >- - -t:RunUnitTests - -p:TF=${{ parameters.targetFramework }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - continueOnError: true + - task: DotNetCoreCLI@2 + displayName: 'Run Flaky Unit Tests ${{parameters.msbuildArchitecture }}' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) + inputs: + command: build + projects: build.proj + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + arguments: >- + -t:TestSqlClientUnit + -p:TestFramework=${{ parameters.targetFramework }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false + ${{ else }}: # x86 + arguments: >- + -t:TestSqlClientUnit + -p:TestFramework=${{ parameters.targetFramework }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false + continueOnError: true - - task: MSBuild@1 + - task: DotNetCoreCLI@2 displayName: 'Run Functional Tests ${{parameters.msbuildArchitecture }}' - condition: succeededOrFailed() + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' + command: build + projects: build.proj ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} + arguments: >- + -t:TestSqlClientFunctional + -p:TestFramework=${{ parameters.targetFramework }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestResultsFolderPath=TestResults ${{ else }}: # x86 - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} + arguments: >- + -t:TestSqlClientFunctional + -p:TestFramework=${{ parameters.targetFramework }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + -p:TestResultsFolderPath=TestResults - - task: MSBuild@1 - condition: succeededOrFailed() + - task: DotNetCoreCLI@2 displayName: 'Run Flaky Functional Tests ${{parameters.msbuildArchitecture }}' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' + command: build + projects: build.proj ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} + arguments: >- + -t:TestSqlClientFunctional + -p:TestFramework=${{ parameters.targetFramework }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false ${{ else }}: # x86 - msbuildArguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} + arguments: >- + -t:TestSqlClientFunctional + -p:TestFramework=${{ parameters.targetFramework }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false continueOnError: true - - task: MSBuild@1 - condition: succeededOrFailed() + - task: DotNetCoreCLI@2 displayName: 'Run Manual Tests ${{parameters.msbuildArchitecture }}' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' + command: build + projects: build.proj ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} + arguments: >- + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestResultsFolderPath=TestResults ${{ else }}: # x86 - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} + arguments: >- + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} -p:DotnetPath=${{ parameters.dotnetx86RootPath }} + -p:TestResultsFolderPath=TestResults retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} - - task: MSBuild@1 - condition: succeededOrFailed() + - task: DotNetCoreCLI@2 displayName: 'Run Flaky Manual Tests ${{parameters.msbuildArchitecture }}' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - solution: build.proj - msbuildArchitecture: ${{parameters.msbuildArchitecture }} - platform: '${{parameters.platform }}' - configuration: '${{parameters.buildConfiguration }}' + command: build + projects: build.proj ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} + arguments: >- + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false ${{ else }}: # x86 - msbuildArguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} + arguments: >- + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} -p:DotnetPath=${{ parameters.dotnetx86RootPath }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false continueOnError: true - ${{ else }}: # Linux or macOS - - ${{if eq(parameters.referenceType, 'Project')}}: - - task: DotNetCoreCLI@2 - displayName: 'Run Unit Tests' - condition: succeededOrFailed() - inputs: - command: custom - projects: build.proj - custom: msbuild - arguments: >- - -t:RunUnitTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Platform=${{ parameters.platform }} - -p:Configuration=${{ parameters.buildConfiguration }} - verbosityRestore: Detailed - verbosityPack: Detailed + - task: DotNetCoreCLI@2 + displayName: 'Run Unit Tests' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) + inputs: + command: build + projects: build.proj + arguments: >- + -t:TestSqlClientUnit + -p:TestFramework=${{ parameters.targetFramework }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestResultsFolderPath=TestResults - - task: DotNetCoreCLI@2 - displayName: 'Run Flaky Unit Tests' - condition: succeededOrFailed() - inputs: - command: custom - projects: build.proj - custom: msbuild - arguments: >- - -t:RunUnitTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} - -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Platform=${{ parameters.platform }} - -p:Configuration=${{ parameters.buildConfiguration }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - verbosityRestore: Detailed - verbosityPack: Detailed - continueOnError: true + - task: DotNetCoreCLI@2 + displayName: 'Run Flaky Unit Tests' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) + inputs: + command: build + projects: build.proj + arguments: >- + -t:TestSqlClientUnit + -p:TestFramework=${{ parameters.targetFramework }} + -p:ReferenceType=${{ parameters.referenceType }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false + continueOnError: true - task: DotNetCoreCLI@2 displayName: 'Run Functional Tests' - condition: succeededOrFailed() + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - command: custom + command: build projects: build.proj - custom: msbuild arguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} + -t:TestSqlClientFunctional + -p:TestFramework=${{ parameters.targetFramework }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Platform=${{ parameters.platform }} -p:Configuration=${{ parameters.buildConfiguration }} - verbosityRestore: Detailed - verbosityPack: Detailed + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestResultsFolderPath=TestResults - task: DotNetCoreCLI@2 - condition: succeededOrFailed() displayName: 'Run Flaky Functional Tests' + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - command: custom + command: build projects: build.proj - custom: msbuild arguments: >- - -t:RunFunctionalTests - -p:TF=${{ parameters.targetFramework }} - -p:TestSet=${{ parameters.testSet }} + -t:TestSqlClientFunctional + -p:TestFramework=${{ parameters.targetFramework }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Platform=${{ parameters.platform }} -p:Configuration=${{ parameters.buildConfiguration }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - verbosityRestore: Detailed - verbosityPack: Detailed + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false continueOnError: true - - task: DotNetCoreCLI@2 displayName: 'Run Manual Tests' - condition: succeededOrFailed() + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - command: custom + command: build projects: build.proj - custom: msbuild arguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:platform=${{ parameters.platform }} -p:Configuration=${{ parameters.buildConfiguration }} - verbosityRestore: Detailed - verbosityPack: Detailed + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestResultsFolderPath=TestResults retryCountOnTaskFailure: ${{parameters.retryCountOnManualTests }} - task: DotNetCoreCLI@2 displayName: 'Run Flaky Manual Tests' - condition: succeededOrFailed() + condition: and(eq(variables['setupSucceeded'], 'true'), succeededOrFailed()) inputs: - command: custom + command: build projects: build.proj - custom: msbuild arguments: >- - -t:RunManualTests - -p:TF=${{ parameters.targetFramework }} + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.targetFramework }} -p:TestSet=${{ parameters.testSet }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:Platform=${{ parameters.platform }} -p:Configuration=${{ parameters.buildConfiguration }} - -p:Filter="category=flaky" - -p:CollectCodeCoverage=false - verbosityRestore: Detailed - verbosityPack: Detailed + -p:PackageVersionSqlClient=${{ parameters.packageVersion }} + -p:PackageVersionSqlServer=${{ parameters.sqlServerPackageVersion }} + -p:TestFilters="category=flaky" + -p:TestResultsFolderPath=TestResults + -p:TestCodeCoverage=false continueOnError: true diff --git a/eng/pipelines/common/templates/steps/update-config-file-step.yml b/eng/pipelines/common/templates/steps/update-config-file-step.yml index b10abdc617..99835f9b1c 100644 --- a/eng/pipelines/common/templates/steps/update-config-file-step.yml +++ b/eng/pipelines/common/templates/steps/update-config-file-step.yml @@ -93,10 +93,6 @@ parameters: type: boolean default: false - - name: SupportsFileStream - type: boolean - default: false - - name: DNSCachingConnString type: string default: '' @@ -121,14 +117,14 @@ parameters: type: boolean default: false - - name: IsAzureSynapse - type: boolean - default: false - - name: ManagedIdentitySupported type: boolean default: true + - name: IsManagedInstance + type: boolean + default: false + - name: WorkloadIdentityFederationServiceConnectionId type: string default: '' @@ -151,9 +147,9 @@ steps: Write-Host "##vso[task.setvariable variable=Password;isSecret=true]$password" displayName: Set Connection String Password - # All properties should be added here, and this template should be used for any manipulation of the config.json file. + # All properties should be added here, and this template should be used for any manipulation of the config.jsonc file. - pwsh: | - $jdata = Get-Content -Raw "config.default.json" | ConvertFrom-Json + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json foreach ($p in $jdata) { $p.TCPConnectionString="${{parameters.TCPConnectionString }}" @@ -188,8 +184,6 @@ steps: $p.DNSCachingConnString="${{parameters.DNSCachingConnString }}" - $p.SupportsFileStream="${{parameters.SupportsFileStream }}" - $p.LocalDbAppName="${{parameters.LocalDbAppName }}" $p.TCPConnectionStringAASSGX="${{parameters.TCPConnectionStringAASSGX }}" @@ -201,20 +195,20 @@ steps: $p.UseManagedSNIOnWindows=[System.Convert]::ToBoolean("${{parameters.UseManagedSNIOnWindows }}") $p.SupportsIntegratedSecurity=[System.Convert]::ToBoolean("${{parameters.SupportsIntegratedSecurity }}") $p.ManagedIdentitySupported=[System.Convert]::ToBoolean("${{parameters.ManagedIdentitySupported }}") - $p.IsAzureSynapse=[System.Convert]::ToBoolean("${{parameters.IsAzureSynapse }}") + $p.IsManagedInstance=[System.Convert]::ToBoolean("${{parameters.IsManagedInstance }}") $p.IsDNSCachingSupportedTR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedTR }}") $p.IsDNSCachingSupportedCR=[System.Convert]::ToBoolean("${{parameters.IsDNSCachingSupportedCR }}") $p.TracingEnabled=[System.Convert]::ToBoolean("${{parameters.TracingEnabled }}") $p.EnclaveEnabled=[System.Convert]::ToBoolean("${{parameters.EnclaveEnabled }}") $p.WorkloadIdentityFederationServiceConnectionId="${{parameters.WorkloadIdentityFederationServiceConnectionId }}" } - $jdata | ConvertTo-Json | Set-Content "config.json" + $jdata | ConvertTo-Json | Set-Content "config.jsonc" workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: 'Update config.json' + displayName: 'Update config.jsonc' - ${{ if eq(parameters.debug, true) }}: - pwsh: | - $jdata = Get-Content -Raw "config.json" | ConvertFrom-Json + $jdata = Get-Content -Raw "config.jsonc" | ConvertFrom-Json foreach ($p in $jdata) { foreach ($prop in $p.PSObject.Properties) @@ -223,4 +217,4 @@ steps: } } workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: '[Debug] Emit config.json' + displayName: '[Debug] Emit config.jsonc' diff --git a/eng/pipelines/common/templates/steps/verify-nuget-package-step.yml b/eng/pipelines/common/templates/steps/verify-nuget-package-step.yml index 6c74c01988..604f1b7894 100644 --- a/eng/pipelines/common/templates/steps/verify-nuget-package-step.yml +++ b/eng/pipelines/common/templates/steps/verify-nuget-package-step.yml @@ -34,7 +34,7 @@ parameters: steps: # Install the .NET SDK, required by the PowerShell script. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self # Invoke the script with the path to the packages to verify. - task: PowerShell@2 diff --git a/eng/pipelines/common/variables/common-variables.yml b/eng/pipelines/common/variables/common-variables.yml new file mode 100644 index 0000000000..5f4532778d --- /dev/null +++ b/eng/pipelines/common/variables/common-variables.yml @@ -0,0 +1,21 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Common variables shared across all pipelines. + +variables: + + ########################################################################## + # "Well-Known" Variables that are ok to use directly, anywhere in the pipeline. + + # The root of our repo. + - name: REPO_ROOT + value: $(Build.SourcesDirectory) + + # This is where our C# projects place their build outputs (see Directory.Build.props + # ). + - name: BUILD_OUTPUT + value: $(REPO_ROOT)/artifacts diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index f9ead686ff..8910ccac33 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -25,6 +25,26 @@ parameters: type: object default: [net8.0, net9.0, net10.0] + # The target frameworks to build and run tests for on Windows, for the + # primary test configurations (local SQL Server 2025 and Azure SQL). + # + # These configurations carry the broadest coverage, so newer runtimes are + # validated here first before being enabled across every configuration. + # + # Note: The driver does not ship a net10.0 target framework, so net10.0 test + # assemblies resolve the net9.0 driver build. Including net10.0 here + # validates the driver running on the .NET 10 runtime. + # + - name: primaryTargetFrameworks + type: object + default: [net462, net8.0, net9.0, net10.0] + + # The target frameworks to build and run tests for on Unix, for the primary + # test configurations (local SQL Server 2025 and Azure SQL). + - name: primaryTargetFrameworksUnix + type: object + default: [net8.0, net9.0, net10.0] + # Netcore Version for Test Utilities - name: netcoreVersionTestUtils type: object @@ -74,14 +94,21 @@ parameters: - Debug - Release + # The name of the general-purpose 1ES pool that most CI jobs run in. + # + # This is the single place in the CI pipelines where the pool name is read + # from a variable group; every stage and job below receives it as a + # parameter so that the value flows down from here. + # + # 'general_purpose_pool_name' is defined in the 'sqlclient-pipeline-config-v1' + # variable group (see /eng/pipelines/libraries/ci-build-variables.yml), for + # both the Public and ADO.Net projects, and names each project's own pool. + # Special-purpose pools, such as the Always Encrypted and ARM64 pools used + # below, are still named directly. + # - name: defaultPoolName type: string - default: $(ci_var_defaultPoolName) - - # True to add a Stress Test stage to the pipeline. - - name: enableStressTests - type: boolean - default: false + default: $(general_purpose_pool_name) # The timeout, in minutes, for each test job. - name: testJobTimeout @@ -93,6 +120,26 @@ parameters: type: boolean default: true + # If true, run manual tests against legacy SQL Server versions (2016, 2017). + # Enabled for CI pipelines; disabled for PR pipelines to keep validation fast. + - name: runLegacySqlTests + type: boolean + default: true + + # If true, run manual tests against SQL Server 2022. + # + # The primary test configurations run against SQL Server 2025, so SQL Server + # 2022 is CI-only coverage. PR pipelines disable it to keep validation fast. + # + - name: runSql22Tests + type: boolean + default: true + + # Build suffix appended to the prerelease tag. PR pipelines pass 'pr', + # CI pipelines pass 'ci'. Official builds leave this empty. + - name: buildSuffix + type: string + - name: dotnetVerbosity type: string default: normal @@ -118,20 +165,43 @@ variables: - name: mdsArtifactsName value: MDS.Artifacts + - name: sqlServerArtifactsName + value: SqlServer.Artifacts + stages: + # Compute all package versions up front. Build and test stages consume the + # compute_versions_ci outputs directly, so no version parameters are passed here. + - template: /eng/pipelines/stages/compute-versions-ci-stage.yml@self + parameters: + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 + buildSuffix: ${{ parameters.buildSuffix }} + # Generate secrets used throughout the pipeline. - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 debug: ${{ parameters.debug }} + # Build the SqlServer package, and publish it to the pipeline artifacts + # under the given artifact name. This runs in parallel with the Secrets + # generation stage since it has no package dependencies. + - template: /eng/pipelines/stages/build-sqlserver-package-ci-stage.yml@self + parameters: + poolName: ${{ parameters.defaultPoolName }} + sqlServerArtifactsName: $(sqlServerArtifactsName) + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + # Build the Logging package, and publish it to the pipeline artifacts # under the given artifact name. This runs in parallel with the Secrets # generation stage since it has no package dependencies. - template: /eng/pipelines/stages/build-logging-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} loggingArtifactsName: $(loggingArtifactsName) - loggingAssemblyFileVersion: $(loggingAssemblyFileVersion) - loggingPackageVersion: $(loggingPackageVersion) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} @@ -143,12 +213,13 @@ stages: # Abstractions has a package dependency on Logging. - template: /eng/pipelines/stages/build-abstractions-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) - abstractionsAssemblyFileVersion: $(abstractionsAssemblyFileVersion) - abstractionsPackageVersion: $(abstractionsPackageVersion) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + loggingArtifactsName: $(loggingArtifactsName) + referenceType: ${{ parameters.referenceType }} # When building Abstractions via packages, we must depend on the Logging # package. ${{ if eq(parameters.referenceType, 'Package') }}: @@ -161,69 +232,58 @@ stages: # - template: /eng/pipelines/stages/build-sqlclient-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) - abstractionsPackageVersion: $(abstractionsPackageVersion) buildConfiguration: ${{ parameters.buildConfiguration }} loggingArtifactsName: $(loggingArtifactsName) - loggingPackageVersion: $(loggingPackageVersion) mdsArtifactsName: $(mdsArtifactsName) - mdsPackageVersion: $(mdsPackageVersion) - akvPackageVersion: $(akvPackageVersion) referenceType: ${{ parameters.referenceType }} + sqlServerArtifactsName: $(sqlServerArtifactsName) SNIVersion: ${{ parameters.SNIVersion }} SNIValidationFeed: ${{ parameters.SNIValidationFeed }} - # When building SqlClient via packages, we must depend on the Abstractions and Logging - # packages. + # When building SqlClient via packages, we must depend on the Abstractions, Logging, + # and SqlServer packages. ${{ if eq(parameters.referenceType, 'Package') }}: additionalDependsOn: - build_abstractions_package_stage - build_logging_package_stage + - build_sqlserver_package_stage # Build the Azure package, and publish it to the pipeline artifacts under the # given artifact name. - template: /eng/pipelines/stages/build-azure-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) - abstractionsPackageVersion: $(abstractionsPackageVersion) azureArtifactsName: $(azureArtifactsName) - azureAssemblyFileVersion: $(azureAssemblyFileVersion) - azurePackageVersion: $(azurePackageVersion) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} # When building via packages, we must depend on the Abstractions, Logging, - # and MDS packages. + # SqlServer, and MDS packages. ${{ if eq(parameters.referenceType, 'Package') }}: additionalDependsOn: - build_abstractions_package_stage - build_logging_package_stage + - build_sqlserver_package_stage - build_sqlclient_package_stage dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + loggingArtifactsName: $(loggingArtifactsName) mdsArtifactsName: $(mdsArtifactsName) - mdsPackageVersion: $(mdsPackageVersion) referenceType: ${{ parameters.referenceType }} + sqlServerArtifactsName: $(sqlServerArtifactsName) # Verify that all NuGet packages comply with Microsoft metadata requirements. # This runs on a Windows agent after all packages have been built and # published as pipeline artifacts. - template: /eng/pipelines/stages/verify-nuget-packages-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-Win25 abstractionsArtifactsName: $(abstractionsArtifactsName) azureArtifactsName: $(azureArtifactsName) loggingArtifactsName: $(loggingArtifactsName) mdsArtifactsName: $(mdsArtifactsName) - - # Run the stress tests, if desired. - - ${{ if eq(parameters.enableStressTests, true) }}: - - template: /eng/pipelines/stages/stress-tests-ci-stage.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - additionalDependsOn: - - build_sqlclient_package_stage - - build_azure_package_stage - mdsArtifactsName: $(mdsArtifactsName) - mdsPackageVersion: $(mdsPackageVersion) - azurePackageVersion: $(azurePackageVersion) - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + sqlServerArtifactsName: $(sqlServerArtifactsName) # Run the MDS and AKV tests. - template: /eng/pipelines/common/templates/stages/ci-run-tests-stage.yml@self @@ -232,19 +292,18 @@ stages: buildConfiguration: ${{ parameters.buildConfiguration }} referenceType: ${{ parameters.referenceType }} abstractionsArtifactsName: $(abstractionsArtifactsName) - abstractionsPackageVersion: $(abstractionsPackageVersion) loggingArtifactsName: $(loggingArtifactsName) - loggingPackageVersion: $(loggingPackageVersion) mdsArtifactsName: $(mdsArtifactsName) - mdsPackageVersion: $(mdsPackageVersion) + sqlServerArtifactsName: $(sqlServerArtifactsName) testJobTimeout: ${{ parameters.testJobTimeout }} # When testing MDS via packages, we must depend on the Abstractions, - # Logging, MDS, and Azure packages. + # Logging, SqlServer, MDS, and Azure packages. ${{ if eq(parameters.referenceType, 'Package') }}: additionalDependsOn: - build_abstractions_package_stage - build_logging_package_stage + - build_sqlserver_package_stage - build_sqlclient_package_stage - build_azure_package_stage @@ -267,6 +326,8 @@ stages: - template: /eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml@self parameters: debug: ${{ parameters.debug }} + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 # We only want to upload coverage results to CodeCov from certain # pipelines. We use the pipeline name (Build.DefinitionName) to # choose. This is a predefined variable that is available at @@ -288,9 +349,66 @@ stages: # Configuration of test jobs. Each entry in this object will become a test job, and the # properties of each entry will be supplied as parameters to the test job template. + # + # The OS architecture is assumed to be x64 unless otherwise noted. + # testConfigurations: - # Windows Server 22 with local SQL Server 2019, x64 build platform. - windows_sql_19_x64: + # SQL Server 2016 and 2017 on Windows Server 2022. + ${{ if eq(parameters.runLegacySqlTests, true) }}: + # Windows Server 22 with local SQL Server 2016. + win22_sql16: + pool: ${{parameters.defaultPoolName }} + images: + Win22_Sql16: ADO-MMS22-SQL16 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: local + operatingSystem: Windows + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: true + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + FileStreamDirectory: $(FileStreamDirectory) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + SQLRootPath: $(SQL16RootPath) + enableLocalDB: true + + # Windows Server 22 with local SQL Server 2017. + win22_sql17: + pool: ${{parameters.defaultPoolName }} + images: + Win22_Sql17: ADO-MMS22-SQL17 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: local + operatingSystem: Windows + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: true + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + FileStreamDirectory: $(FileStreamDirectory) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + SQLRootPath: $(SQL17RootPath) + enableLocalDB: true + + # Windows Server 22 with local SQL Server 2019. + win22_sql19: pool: ${{parameters.defaultPoolName }} images: Win22_Sql19: ADO-MMS22-SQL19 @@ -301,7 +419,7 @@ stages: useManagedSNI: ${{parameters.useManagedSNI }} configSqlFor: local operatingSystem: Windows - # config.json properties + # config.jsonc properties configProperties: TCPConnectionString: $(SQL_TCP_CONN_STRING) NPConnectionString: $(SQL_NP_CONN_STRING) @@ -316,8 +434,8 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2019, x86 build platform. - windows_sql_19_x86: + # Windows Server 22 with local SQL Server 2019, x86. + win22_sql19_x86: pool: ${{parameters.defaultPoolName }} images: Win22_Sql19_x86: ADO-MMS22-SQL19 @@ -343,12 +461,12 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022, x64 build platform. - windows_sql_22_x64: + # Windows Server 25 with local SQL Server 2025. + win25_sql25: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22: ADO-MMS22-SQL22 - TargetFrameworks: ${{parameters.targetFrameworks }} + Win25_Sql25: ADO-MMS25-SQL25 + TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} testSets: ${{parameters.testSets }} @@ -366,14 +484,14 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022, x86 build platform. - windows_sql_22_x86: + # Windows Server 25 with local SQL Server 2025, x86. + win25_sql25_x86: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_x86: ADO-MMS22-SQL22 + Win25_Sql25_x86: ADO-MMS25-SQL25 TargetFrameworks: [net462, net8.0, net9.0] netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -393,14 +511,14 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) x86TestTargetFrameworks: [net462, net8.0, net9.0] - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022 Named Instance, x64 build platform. - windows_sql_22_named_instance: + # Windows Server 25 with local SQL Server 2025 Named Instance. + win25_sql25_named_instance: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_Named_Instance: ADO-MMS22-SQL22-WITH-NAMED-INSTANCE + Win25_Sql25_Named_Instance: ADO-MMS25-SQL25-WITH-NAMED-INSTANCE TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -412,16 +530,15 @@ stages: TCPConnectionString: $(SQL_TCP_INSTANCE_CONN_STRING) NPConnectionString: $(SQL_NP_INSTANCE_CONN_STRING) SupportsIntegratedSecurity: true - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) instanceName: $(NamedInstance) - # Windows Server 2022 and Windows 11, x64 build platform, with Azure SQL Server. - windows_azure_sql: + # Windows Server 2025, x64 build platform, with Azure SQL Server. + win25_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Win22_Azure_Sql: ADO-MMS22-SQL19 - Win11_Azure_Sql: ADO-CI-Win11 - TargetFrameworks: ${{parameters.targetFrameworks }} + Win25_Azure_Sql: ADO-Win25 + TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} testSets: ${{parameters.testSets }} @@ -446,12 +563,38 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + # Windows 11, x64 build platform, with Azure SQL Server. + win11_azure_sql: + pool: ${{parameters.defaultPoolName }} + images: + Win11_Azure_Sql: ADO-CI-Win11 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: azure + operatingSystem: Windows + configProperties: + TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) + NPConnectionString: $(AZURE_DB_NP_CONN_STRING) + AADAuthorityURL: $(AADAuthorityURL) + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: + AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR) + AADServicePrincipalSecret: $(AADServicePrincipalSecret) + AADServicePrincipalId: $(AADServicePrincipalId) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: false + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + # Windows 11 on ARM64 with Azure SQL Server. - windows_azure_arm64_sql: + win11_azure_sql_arm64: pool: ADO-CI-PUBLIC-ARM64-1ES-EUS-POOL images: - Win11_ARM64_Azure_Sql: ADO-WIN11-ARM64 - isArm64: true + Win11_Azure_Sql_ARM64: ADO-WIN11-ARM64 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -474,13 +617,12 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 20 and 22 with local SQL Server 2022, x64 build platform. - linux_ub20_22_sql_22: + # Linux Ubuntu 24 with local SQL Server 2025. + linux_ub24_sql25: pool: ${{parameters.defaultPoolName }} images: - Ubuntu20_Sql22: ADO-UB20-SQL22 - Ubuntu22_Sql22: ADO-UB22-SQL22 - TargetFrameworks: ${{parameters.targetFrameworksUnix }} + Ubuntu24_Sql25: ADO-UB24-SQL25 + TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] testSets: ${{parameters.testSets }} @@ -498,12 +640,68 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - # Linux Ubuntu 22 with Azure SQL Server, x64 build platform. - linux_azure_sql: + # SQL Server 2022 coverage, on Windows and Linux. + # + # The primary configurations run against SQL Server 2025, so these keep + # SQL Server 2022 in the matrix. They are grouped under a single + # conditional so that PR pipelines can opt out of them as a unit. + ${{ if eq(parameters.runSql22Tests, true) }}: + # Windows Server 22 with local SQL Server 2022. + win22_sql22: + pool: ${{parameters.defaultPoolName }} + images: + Win22_Sql22: ADO-MMS22-SQL22 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: local + operatingSystem: Windows + # config.jsonc properties + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: true + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + FileStreamDirectory: $(FileStreamDirectory) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + SQLRootPath: $(SQL22RootPath) + enableLocalDB: true + + # Linux Ubuntu 22 with local SQL Server 2022. + linux_ub22_sql22: + pool: ${{parameters.defaultPoolName }} + images: + Ubuntu22_Sql22: ADO-UB22-SQL22 + TargetFrameworks: ${{parameters.targetFrameworksUnix }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: [AnyCPU] + testSets: ${{parameters.testSets }} + useManagedSNI: [true] + configSqlFor: local + operatingSystem: Linux + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: false + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + + # Linux Ubuntu 24 with Azure SQL Server. + linux_ub24_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Ubuntu22_Azure_Sql: ADO-UB22-SQL22 - TargetFrameworks: ${{parameters.targetFrameworksUnix }} + Ubuntu24_Azure_Sql: ADO-UB24 + TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] testSets: ${{parameters.testSets }} @@ -525,12 +723,11 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # macOS with local SQL Server 2022, x64 build platform. - mac_sql_22: + # macOS with local SQL Server 2025 (docker), x64 build platform. + mac_sql25: pool: Azure Pipelines - hostedPool: true images: - MacOSLatest_Sql22: macos-latest + MacOSLatest_Sql25: macos-latest TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] @@ -551,11 +748,11 @@ stages: # Only run these tests if explicitly enabled, and if we're not a forked repo (which won't # have access to the necessary Library secrets). ${{ if and(eq(parameters.runAlwaysEncryptedTests, true), eq(variables['System.PullRequest.IsFork'], 'False')) }}: - # Windows Server 22 with remote Enclave-enabled SQL Server 2019, x64 build platform. - windows_enclave_sql: + # Windows Server 22 with remote Enclave-enabled SQL Server 2019. + win22_enclave_sql19: pool: ADO-CI-AE-1ES-Pool images: - Win22_Enclave_Sql19: ADO-MMS22-SQL19 + Win22_Enclave_Sql19: ADO-Win25 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -579,11 +776,11 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 22 with remote Enclave-enabled SQL Server 2019, x64 build platform. - linux_enclave_sql: + # Linux Ubuntu 24 with remote Enclave-enabled SQL Server 2019. + linux_ub24_enclave_sql19: pool: ADO-CI-AE-1ES-Pool images: - Ubuntu20_Enclave_Sql19: ADO-UB22-Sql22 + Ubuntu24_Enclave_Sql19: ADO-UB24 TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] diff --git a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml index 262c2b909a..0b84885e2d 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml @@ -14,18 +14,20 @@ # It runs via CI push triggers and schedules and uses the Release build # configuration: # -# - Commits to GitHub main -# - Commits to ADO internal/main -# - Weekdays at 03:00 UTC on GitHub main -# - Thursdays at 07:00 UTC on ADO internal/main +# - GitHub CI push trigger disabled due to limited resources +# - Commits to the ADO internal/release/7.1 branch +# - Daily at 11:00 UTC on GitHub release/7.1 +# - Daily at 19:00 UTC on ADO internal/release/7.1 # -# GOTCHA: This pipeline definition is triggered by GitHub _and_ ADO CI. We are -# able to define different triggers and schedules using branch filters: +# GOTCHA: This pipeline definition is registered with GitHub _and_ ADO CI. We +# are able to define different triggers and schedules using branch filters: # -# - Only the GitHub repo has a 'main' branch, so its presence indicates that -# the pipeline run was triggered via GitHub. +# - The GitHub registration uses the release/7.1 branch filters. Only its +# schedule is active; the CI push trigger is commented out below due to +# limited resources. # -# - Only the ADO repo has an 'internal/main' branch. +# - The ADO registration uses the internal/release/7.1 branch filters, for +# both the push trigger and the schedule. # # Changes are batched together to ensure that the pipline never runs # concurrently. @@ -41,6 +43,9 @@ # https://sqlclientdrivers.visualstudio.com/ADO.Net/_build?definitionId=1933 # Set the pipeline run name to the day-of-year and the daily run counter. +# This becomes BuildNumber, used as the revision component of FileVersion +# (Major.Minor.Patch.Revision). Revision must fit in a 16-bit unsigned int (max 65535). +# Format $(DayOfYear)$(Rev:rr) yields e.g. 15401 (day 154, run 01) — always < 65535. name: $(DayOfYear)$(Rev:rr) # Do not trigger this pipeline for PRs. @@ -55,29 +60,31 @@ trigger: branches: include: - # GitHub main branch. - - main + # GitHub release branch. + # + # GOTCHA: Currently disabled due to limited resources. + #- release/7.1 - # ADO internal/main branch. - - internal/main + # ADO release branch. + - internal/release/7.1 -# Trigger this pipline on a schedule. +# Trigger this pipeline on a schedule. schedules: - # GitHub main on weekdays - - cron: '0 3 * * Mon-Fri' - displayName: Weekday Run (Release Config) + # GitHub release/7.1 daily. + - cron: '0 11 * * *' + displayName: 7.1 GitHub Package-Reference Daily Run (11:00 UTC) branches: include: - - main + - release/7.1 always: true - # ADO internal/main on Thursdays. - - cron: '0 7 * * Thu' - displayName: Thursday Run (Release Config) + # ADO internal/release/7.1 daily. + - cron: '0 19 * * *' + displayName: 7.1 ADO Package-Reference Daily Run (19:00 UTC) branches: include: - - internal/main + - internal/release/7.1 always: true # Pipeline parameters, visible in the Azure DevOps UI. @@ -103,12 +110,6 @@ parameters: type: boolean default: false - # True to run the stress tests stage. - - name: enableStressTests - displayName: Enable Stress Tests - type: boolean - default: false - # The target frameworks to build and run tests for on Windows. # # These are _not_ the target frameworks to build the driver packages for. @@ -170,12 +171,17 @@ extends: parameters: buildConfiguration: ${{ parameters.buildConfiguration }} buildPlatforms: ${{ parameters.buildPlatforms }} + buildSuffix: 'ci' referenceType: Package debug: ${{ parameters.debug }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - enableStressTests: ${{ parameters.enableStressTests }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml index a5a78f0f8b..7c25f28e60 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml @@ -14,18 +14,17 @@ # It runs via CI push triggers and schedules and uses the Release build # configuration: # -# - Commits to GitHub main -# - Commits to ADO internal/main -# - Weekdays at 01:00 UTC on GitHub main -# - Thursdays at 05:00 UTC on ADO internal/main +# - Commits to the GitHub release/7.1 branch +# - Commits to the ADO internal/release/7.1 branch +# - Daily at 09:00 UTC on GitHub release/7.1 +# - Daily at 17:00 UTC on ADO internal/release/7.1 # # GOTCHA: This pipeline definition is triggered by GitHub _and_ ADO CI. We are # able to define different triggers and schedules using branch filters: # -# - Only the GitHub repo has a 'main' branch, so its presence indicates that -# the pipeline run was triggered via GitHub. +# - The GitHub registration uses the release/7.1 branch filters. # -# - Only the ADO repo has an 'internal/main' branch. +# - The ADO registration uses the internal/release/7.1 branch filters. # # Changes are batched together to ensure that the pipline never runs # concurrently. @@ -41,6 +40,9 @@ # https://sqlclientdrivers.visualstudio.com/ADO.Net/_build?definitionId=1825 # Set the pipeline run name to the day-of-year and the daily run counter. +# This becomes BuildNumber, used as the revision component of FileVersion +# (Major.Minor.Patch.Revision). Revision must fit in a 16-bit unsigned int (max 65535). +# Format $(DayOfYear)$(Rev:rr) yields e.g. 15401 (day 154, run 01) — always < 65535. name: $(DayOfYear)$(Rev:rr) # Do not trigger this pipeline for PRs. @@ -55,29 +57,29 @@ trigger: branches: include: - # GitHub main branch. - - main + # GitHub release branch. + - release/7.1 - # ADO internal/main branch. - - internal/main + # ADO release branch. + - internal/release/7.1 -# Trigger this pipline on a schedule. +# Trigger this pipeline on a schedule. schedules: - # GitHub main on weekdays - - cron: '0 1 * * Mon-Fri' - displayName: Weekday Run (Release Config) + # GitHub release/7.1 daily. + - cron: '0 9 * * *' + displayName: 7.1 GitHub Project-Reference Daily Run (09:00 UTC) branches: include: - - main + - release/7.1 always: true - # ADO internal/main on Thursdays. - - cron: '0 5 * * Thu' - displayName: Thursday Run (Release Config) + # ADO internal/release/7.1 daily. + - cron: '0 17 * * *' + displayName: 7.1 ADO Project-Reference Daily Run (17:00 UTC) branches: include: - - internal/main + - internal/release/7.1 always: true # Pipeline parameters, visible in the Azure DevOps UI. @@ -103,19 +105,14 @@ parameters: type: boolean default: false - # True to run the stress tests stage. - - name: enableStressTests - displayName: Enable Stress Tests - type: boolean - default: false - # The target frameworks to build and run tests for on Windows. # # These are _not_ the target frameworks to build the driver packages for. # # Note: We are excluding .NET 10.0 here to avoid consuming too many resources - # during PR pipeline runs, and until we update our 1ES images to include - # Visual Studio 2026 (18.0) whose MSBuild SDK supports .NET 10. + # across every SQL Server image. .NET 10.0 coverage is provided by the + # primaryTargetFrameworks parameter below, which applies to the SQL Server + # 2025 and Azure SQL configurations. # - name: targetFrameworks displayName: Target Frameworks on Windows @@ -127,14 +124,38 @@ parameters: # These are _not_ the target frameworks to build the driver packages for. # # Note: We are excluding .NET 10.0 here to avoid consuming too many resources - # during PR pipeline runs, and until we update our 1ES images to include - # Visual Studio 2026 (18.0) whose MSBuild SDK supports .NET 10. + # across every SQL Server image. .NET 10.0 coverage is provided by the + # primaryTargetFrameworksUnix parameter below, which applies to the SQL + # Server 2025 and Azure SQL configurations. # - name: targetFrameworksUnix displayName: Target Frameworks on Unix type: object default: [net8.0, net9.0] + # The target frameworks used by the primary test configurations (local SQL + # Server 2025 and Azure SQL) on Windows. + # + # net10.0 is included here so that unit, functional, and manual tests get + # .NET 10 coverage without paying for it on every legacy SQL Server image. + # + # Note: The driver itself does not ship a net10.0 target framework, so the + # net10.0 test assemblies resolve the net9.0 driver build. These jobs + # therefore validate the driver running on the .NET 10 runtime, rather than + # a net10.0 build of the driver. + # + - name: primaryTargetFrameworks + displayName: Target Frameworks on Windows (SQL 2025 and Azure SQL) + type: object + default: [net462, net8.0, net9.0, net10.0] + + # The target frameworks used by the primary test configurations (local SQL + # Server 2025 and Azure SQL) on Unix. + - name: primaryTargetFrameworksUnix + displayName: Target Frameworks on Unix (SQL 2025 and Azure SQL) + type: object + default: [net8.0, net9.0, net10.0] + # The timeout, in minutes, for each test job. - name: testJobTimeout displayName: Test job timeout (in minutes) @@ -170,12 +191,14 @@ extends: parameters: buildConfiguration: ${{ parameters.buildConfiguration }} buildPlatforms: ${{ parameters.buildPlatforms }} + buildSuffix: 'ci' referenceType: Project debug: ${{ parameters.debug }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - enableStressTests: ${{ parameters.enableStressTests }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + primaryTargetFrameworks: ${{ parameters.primaryTargetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.primaryTargetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/github-sync-pipeline.yml b/eng/pipelines/github-sync-pipeline.yml new file mode 100644 index 0000000000..5e3d7c1d19 --- /dev/null +++ b/eng/pipelines/github-sync-pipeline.yml @@ -0,0 +1,92 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This pipeline synchronizes the GitHub dotnet/SqlClient repository's release/7.1 +# branch into the internal ADO repository by: +# +# 1. Fetching the latest commits from GitHub release/7.1. +# 2. Pushing them to a dev/autosync/github-release-7.1 branch in ADO. +# 3. Creating (or updating) a pull request targeting internal/release/7.1. +# +# It runs on a daily schedule at 15:00 UTC and can also be triggered manually. +# The pipeline does not auto-complete the PR — changes must be reviewed and +# merged manually. +# +# Prerequisites: +# +# - The build service identity must have "Contribute", "Create branch", and +# "Contribute to pull requests" permissions on the ADO repository. +# +# This pipeline definition is mapped to the following Azure DevOps pipeline: +# +# - GitHub Sync in the ADO.Net project: +# +# https://sqlclientdrivers.visualstudio.com/ADO.Net/_build?definitionId=2263 + +# Set the pipeline run name to the calendar date (yyyyMMdd) and the daily run counter. +name: Sync-$(Date:yyyyMMdd)$(Rev:.r) + +# Do not trigger this pipeline on commits or PRs. +trigger: none +pr: none + +# Trigger this pipeline on a daily schedule. +schedules: + - cron: '0 15 * * *' + displayName: 7.1 Daily GitHub Sync (15:00 UTC) + branches: + include: + - internal/release/7.1 + always: true + +# Pipeline parameters, visible in the Azure DevOps UI. +parameters: + + # The GitHub branch to sync from. + - name: githubBranch + displayName: GitHub Branch + type: string + default: release/7.1 + + # The ADO target branch to create the PR against. + - name: targetBranch + displayName: ADO Target Branch + type: string + default: internal/release/7.1 + +variables: + - template: /eng/pipelines/libraries/ci-build-variables.yml@self + +jobs: + - job: SyncGitHub + displayName: Sync GitHub release/7.1 to ADO internal/release/7.1 + pool: + name: $(general_purpose_pool_name) + demands: + - imageOverride -equals ADO-UB24 + + steps: + # Check out the ADO repo with full history so we can compare branches. + - checkout: self + persistCredentials: true + fetchDepth: 0 + + # Run the sync script. + - task: PowerShell@2 + displayName: Sync GitHub release/7.1 to ADO + inputs: + filePath: $(Build.SourcesDirectory)/eng/pipelines/scripts/Sync-GitHubToAdo.ps1 + arguments: >- + -GitHubRepoUrl "https://github.com/dotnet/SqlClient.git" + -GitHubBranch "${{ parameters.githubBranch }}" + -TargetBranch "${{ parameters.targetBranch }}" + -SyncBranchName "dev/autosync/github-${{ replace(parameters.githubBranch, '/', '-') }}" + -AdoOrgUrl "$(System.CollectionUri)" + -AdoProject "$(System.TeamProject)" + -AdoRepoName "$(Build.Repository.Name)" + pwsh: true + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml index 9db1c05696..3b01faddc5 100644 --- a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml @@ -17,12 +17,9 @@ parameters: type: string default: Abstractions.Artifact - # The assembly file version to stamp into the Abstractions DLLs. - - name: abstractionsAssemblyFileVersion - type: string - - # The version to apply to the Abstractions NuGet package and its assemblies. - - name: abstractionsPackageVersion + # The version to apply to the Abstractions NuGet package and its assemblies. Every package in the + # SqlClient family shares this version. + - name: packageVersion type: string # The type of build to test (Release or Debug) @@ -53,6 +50,35 @@ parameters: - detailed - diagnostic + # The name of the Logging pipeline artifacts to download. + # + # This is used when the referenceType is 'Package'. + - name: loggingArtifactsName + type: string + default: Logging.Artifacts + + # The C# project reference type to use when building and packing the packages. + - name: referenceType + type: string + default: Project + values: + # Reference sibling packages as NuGet packages. + - Package + # Reference sibling packages as C# projects. + - Project + + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_abstractions_package_job @@ -61,8 +87,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -78,9 +106,7 @@ jobs: # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. # This is defined with a non-standard Platform of 'AnyCPU', and will fail - # the builds if left defined. The stress tests solution does not require - # any specific Platform, and so its solution file doesn't support any - # non-standard platforms. + # the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. @@ -104,21 +130,52 @@ jobs: - pwsh: 'Get-ChildItem Env: | Sort-Object Name' displayName: '[Debug] Print Environment Variables' + # For Package reference builds, we must first download the dependency + # package artifacts. + - ${{ if eq(parameters.referenceType, 'Package') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Logging Package Artifacts + inputs: + artifactName: ${{ parameters.loggingArtifactsName }} + targetPath: $(Build.SourcesDirectory)/packages + # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} # Create the NuGet packages. - - task: DotNetCoreCLI@2 - displayName: Create NuGet Package - inputs: - command: pack - packagesToPack: $(project) - configurationToPack: ${{ parameters.buildConfiguration }} - packDirectory: $(dotnetPackagesDir) - verbosityToPack: ${{ parameters.dotnetVerbosity }} - buildProperties: AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }};AbstractionsAssemblyFileVersion=${{ parameters.abstractionsAssemblyFileVersion }} + # + # When referenceType is Package, we must pass ReferenceType and the + # dependency version so that Directory.Packages.props applies version + # ranges to sibling package dependencies. + - ${{ if eq(parameters.referenceType, 'Package') }}: + - task: DotNetCoreCLI@2 + displayName: Create NuGet Package + inputs: + command: pack + packagesToPack: $(project) + configurationToPack: ${{ parameters.buildConfiguration }} + packDirectory: $(dotnetPackagesDir) + verbosityToPack: ${{ parameters.dotnetVerbosity }} + # BuildNumber supplies the revision component of FileVersion; without + # it the assembly is stamped Major.Minor.Patch.0 (see Project branch). + buildProperties: SqlClientPackageVersion=${{ parameters.packageVersion }};ReferenceType=Package;BuildNumber=$(Build.BuildNumber) + + - ${{ else }}: + - task: DotNetCoreCLI@2 + displayName: Create NuGet Package + inputs: + command: pack + packagesToPack: $(project) + configurationToPack: ${{ parameters.buildConfiguration }} + packDirectory: $(dotnetPackagesDir) + verbosityToPack: ${{ parameters.dotnetVerbosity }} + # BuildNumber supplies the revision component of FileVersion + # (Major.Minor.Patch.Revision). Without it, FileVersionBuildNumber + # defaults to 0 and the assembly is stamped Major.Minor.Patch.0, + # inconsistent with the MDS/AKV packages that pass it. + buildProperties: SqlClientPackageVersion=${{ parameters.packageVersion }};BuildNumber=$(Build.BuildNumber) # Publish the NuGet packages as a named pipeline artifact. - task: PublishPipelineArtifact@1 diff --git a/eng/pipelines/jobs/pack-azure-package-ci-job.yml b/eng/pipelines/jobs/pack-azure-package-ci-job.yml index 95853d43e0..884d4a5c32 100644 --- a/eng/pipelines/jobs/pack-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-azure-package-ci-job.yml @@ -26,23 +26,14 @@ parameters: type: string default: Logging.Artifacts - # The Abstractions package verion to depend on. - # - # This is used when the referenceType is 'Package'. - - name: abstractionsPackageVersion - type: string - # The name of the pipeline artifacts to publish. - name: azureArtifactsName type: string default: Azure.Artifacts - # The assembly file version to stamp into the Azure DLLs. - - name: azureAssemblyFileVersion - type: string - - # The version to apply to the NuGet package and DLLs. - - name: azurePackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging). They + # all share this version. + - name: packageVersion type: string # The type of build to test (Release or Debug) @@ -82,6 +73,18 @@ parameters: # Reference sibling packages as C# projects. - Project + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_azure_package_job @@ -90,8 +93,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -107,9 +112,7 @@ jobs: # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public # project. This is defined with a non-standard Platform of 'AnyCPU', and - # will fail the builds if left defined. The stress tests solution does - # not require any specific Platform, and so its solution file doesn't - # support any non-standard platforms. + # will fail the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. @@ -148,20 +151,42 @@ jobs: targetPath: $(Build.SourcesDirectory)/packages # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} # Create the NuGet packages. - - task: DotNetCoreCLI@2 - displayName: Create NuGet Package - inputs: - command: pack - packagesToPack: $(project) - configurationToPack: ${{ parameters.buildConfiguration }} - packDirectory: $(dotnetPackagesDir) - verbosityToPack: ${{ parameters.dotnetVerbosity }} - buildProperties: AzurePackageVersion=${{ parameters.azurePackageVersion }};AzureAssemblyFileVersion=${{ parameters.azureAssemblyFileVersion }} + # + # When referenceType is Package, we must pass ReferenceType and the + # dependency versions so that Directory.Packages.props applies version + # ranges to sibling package dependencies. + - ${{ if eq(parameters.referenceType, 'Package') }}: + - task: DotNetCoreCLI@2 + displayName: Create NuGet Package + inputs: + command: pack + packagesToPack: $(project) + configurationToPack: ${{ parameters.buildConfiguration }} + packDirectory: $(dotnetPackagesDir) + verbosityToPack: ${{ parameters.dotnetVerbosity }} + # BuildNumber supplies the revision component of FileVersion; without + # it the assembly is stamped Major.Minor.Patch.0 (see Project branch). + buildProperties: SqlClientPackageVersion=${{ parameters.packageVersion }};ReferenceType=Package;BuildNumber=$(Build.BuildNumber) + + - ${{ else }}: + - task: DotNetCoreCLI@2 + displayName: Create NuGet Package + inputs: + command: pack + packagesToPack: $(project) + configurationToPack: ${{ parameters.buildConfiguration }} + packDirectory: $(dotnetPackagesDir) + verbosityToPack: ${{ parameters.dotnetVerbosity }} + # BuildNumber supplies the revision component of FileVersion + # (Major.Minor.Patch.Revision). Without it, FileVersionBuildNumber + # defaults to 0 and the assembly is stamped Major.Minor.Patch.0, + # inconsistent with the MDS/AKV packages that pass it. + buildProperties: SqlClientPackageVersion=${{ parameters.packageVersion }};BuildNumber=$(Build.BuildNumber) # Publish the NuGet packages as a named pipeline artifact. - task: PublishPipelineArtifact@1 diff --git a/eng/pipelines/jobs/pack-logging-package-ci-job.yml b/eng/pipelines/jobs/pack-logging-package-ci-job.yml index dd748cb4b2..1adcf3ac1d 100644 --- a/eng/pipelines/jobs/pack-logging-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-logging-package-ci-job.yml @@ -17,12 +17,9 @@ parameters: type: string default: Logging.Artifacts - # The assembly file version to stamp into the Logging DLLs. - - name: loggingAssemblyFileVersion - type: string - - # The version to apply to the Logging NuGet package and its assemblies. - - name: loggingPackageVersion + # The version to apply to the Logging NuGet package and its assemblies. Every package in the + # SqlClient family shares this version. + - name: packageVersion type: string # The type of build to test (Release or Debug) @@ -53,6 +50,18 @@ parameters: - detailed - diagnostic + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_logging_package_job @@ -61,8 +70,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -94,7 +105,7 @@ jobs: displayName: '[Debug] Print Environment Variables' # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} @@ -107,7 +118,11 @@ jobs: configurationToPack: ${{ parameters.buildConfiguration }} packDirectory: $(dotnetPackagesDir) verbosityToPack: ${{ parameters.dotnetVerbosity }} - buildProperties: LoggingPackageVersion=${{ parameters.loggingPackageVersion }};LoggingAssemblyFileVersion=${{ parameters.loggingAssemblyFileVersion }} + # BuildNumber supplies the revision component of FileVersion + # (Major.Minor.Patch.Revision). Without it, FileVersionBuildNumber + # defaults to 0 and the assembly is stamped Major.Minor.Patch.0, + # inconsistent with the MDS/AKV packages that pass it. + buildProperties: SqlClientPackageVersion=${{ parameters.packageVersion }};BuildNumber=$(Build.BuildNumber) # Publish the NuGet packages as a named pipeline artifact. - task: PublishPipelineArtifact@1 diff --git a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml new file mode 100644 index 0000000000..54938389a5 --- /dev/null +++ b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml @@ -0,0 +1,130 @@ +################################################################################ +# Licensed to the .NET Foundation under one or more agreements. The .NET +# Foundation licenses this file to you under the MIT license. See the LICENSE +# file in the project root for more information. +################################################################################ + +# This job packs the Microsoft.SqlServer.Server package into NuGet and symbols +# packages and publishes them as a named pipeline artifact. +# +# This template defines a job named 'pack_sqlserver_package_job' that can be +# depended on by downstream jobs. + +parameters: + + # The type of build to produce (Release or Debug) + - name: buildConfiguration + type: string + values: + - Release + - Debug + + # True to emit debug information and steps. + - name: debug + type: boolean + default: false + + # The name of the pipeline artifacts to publish. + - name: sqlServerArtifactsName + type: string + default: SqlServer.Artifacts + + # The version to apply to the SqlServer NuGet package and its assemblies. + - name: sqlServerPackageVersion + type: string + + # The list of upstream jobs to depend on. + - name: dependsOn + type: object + default: [] + + # The verbosity level for the dotnet CLI commands. + - name: dotnetVerbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + +jobs: + + - job: pack_sqlserver_package_job + displayName: Pack SqlServer Package + + dependsOn: ${{ parameters.dependsOn }} + + pool: + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} + + variables: + + # The SqlServer project file to use for all dotnet CLI commands. + - name: project + value: src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj + + # The directory where the NuGet packages will be staged before being + # published as pipeline artifacts. + - name: dotnetPackagesDir + value: $(Build.StagingDirectory)/dotnetPackages + + # Explicitly unset the $PLATFORM environment variable that is set by the + # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. + - name: Platform + value: '' + + # Do the same for $CONFIGURATION since we explicitly set it using our + # 'buildConfiguration' parameter, and we don't want the environment to + # override us. + - name: Configuration + value: '' + + steps: + + # Emit environment variables if debug is enabled. + - ${{ if eq(parameters.debug, true) }}: + - pwsh: 'Get-ChildItem Env: | Sort-Object Name' + displayName: '[Debug] Print Environment Variables' + + # Install the .NET SDK. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + debug: ${{ parameters.debug }} + + # Create the NuGet packages. + - task: DotNetCoreCLI@2 + displayName: Create NuGet Package + inputs: + command: pack + packagesToPack: $(project) + configurationToPack: ${{ parameters.buildConfiguration }} + packDirectory: $(dotnetPackagesDir) + verbosityToPack: ${{ parameters.dotnetVerbosity }} + # BuildNumber supplies the revision component of FileVersion + # (Major.Minor.Patch.Revision). Without it, FileVersionBuildNumber + # defaults to 0 and the assembly is stamped Major.Minor.Patch.0. + buildProperties: SqlServerPackageVersion=${{ parameters.sqlServerPackageVersion }};BuildNumber=$(Build.BuildNumber) + + - task: PublishPipelineArtifact@1 + displayName: Publish Pipeline Artifact + inputs: + targetPath: $(dotnetPackagesDir) + artifactName: ${{ parameters.sqlServerArtifactsName }} + publishLocation: pipeline diff --git a/eng/pipelines/jobs/stress-tests-ci-job.yml b/eng/pipelines/jobs/stress-tests-ci-job.yml deleted file mode 100644 index 7477295f54..0000000000 --- a/eng/pipelines/jobs/stress-tests-ci-job.yml +++ /dev/null @@ -1,194 +0,0 @@ -################################################################################ -# Licensed to the .NET Foundation under one or more agreements. The .NET -# Foundation licenses this file to you under the MIT license. See the LICENSE -# file in the project root for more information. -################################################################################ - -# This job builds and runs stress tests against an MDS NuGet package available -# as a pipeline artifact. -# -# The stress tests are located here: -# -# src/Microsoft.Data.SqlClient/tests/StressTests -# -# This template defines a job named 'run_stress_tests_job_' that can be -# depended on by downstream jobs. - -parameters: - # The suffix to append to the job name. - - name: jobNameSuffix - type: string - default: '' - - # The prefix to prepend to the job's display name: - # - # [] Run Stress Tests - # - - name: displayNamePrefix - type: string - default: '' - - # The name of the Azure Pipelines pool to use. - - name: poolName - type: string - default: '' - - # The pool VM image to use. - - name: vmImage - type: string - default: '' - - # The pipeline step to run to configure SQL Server. - - name: sqlSetupStep - type: step - - # The name of the MDS pipeline artifacts to download. - - name: mdsArtifactsName - type: string - default: '' - - # The solution file to restore/build. - - name: solution - type: string - default: '' - - # The test project to run. - - name: testProject - type: string - default: '' - - # dotnet CLI arguments for the restore step. - - name: restoreArguments - type: string - default: '' - - # dotnet CLI arguments for the build and run steps. - - name: buildArguments - type: string - default: '' - - # The list of .NET runtimes to test against. - - name: netTestRuntimes - type: object - default: [] - - # The list of .NET Framework runtimes to test against. - - name: netFrameworkTestRuntimes - type: object - default: [] - - # The stress test config file contents to write to the config file. - - name: configContent - type: string - default: '' - -jobs: -- job: run_stress_tests_job_${{ parameters.jobNameSuffix }} - displayName: '[${{ parameters.displayNamePrefix }}] Run Stress Tests' - pool: - name: ${{ parameters.poolName }} - ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} - ${{ else }}: - demands: - - imageOverride -equals ${{ parameters.vmImage }} - - variables: - # Stress test command-line arguments. - - name: testArguments - value: -a SqlClient.Stress.Tests -console - - # Explicitly unset the $PLATFORM environment variable that is set by the - # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. - # This is defined with a non-standard Platform of 'AnyCPU', and will fail - # the builds if left defined. The stress tests solution does not require - # any specific Platform, and so its solution file doesn't support any - # non-standard platforms. - # - # Note that Azure Pipelines will inject this variable as PLATFORM into the - # environment of all tasks in this job. - # - # See: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch - # - - name: Platform - value: '' - - # Do the same for $CONFIGURATION since we explicitly set it using our - # 'buildConfiguration' parameter, and we don't want the environment to - # override us. - - name: Configuration - value: '' - - steps: - - # Install the .NET SDK and Runtimes. - - template: /eng/pipelines/steps/install-dotnet.yml@self - parameters: - runtimes: [8.x, 9.x] - - # Download the pipeline artifact that contains the MDS package to test. - - task: DownloadPipelineArtifact@2 - displayName: Download MDS Artifacts - inputs: - artifactName: ${{ parameters.mdsArtifactsName }} - targetPath: $(Build.SourcesDirectory)/packages - - # Setup the local SQL Server. - - ${{ parameters.sqlSetupStep }} - - # We use the 'custom' command because the DotNetCoreCLI@2 task doesn't support - # all of our argument combinations for the different build steps. - - # Restore the solution. - - task: DotNetCoreCLI@2 - displayName: Restore Solution - inputs: - command: custom - custom: restore - projects: ${{ parameters.solution }} - arguments: ${{ parameters.restoreArguments }} - - # Build the solution. - - task: DotNetCoreCLI@2 - displayName: Build Solution - inputs: - command: custom - custom: build - projects: ${{ parameters.solution }} - arguments: ${{ parameters.buildArguments }} --no-restore - - # Write the config file. - - task: PowerShell@2 - displayName: Write Config File - inputs: - pwsh: true - targetType: inline - script: | - # Capture the multi-line JSON content into a variable. - $content = @" - ${{ parameters.configContent }} - "@ - - # Write the JSON content to the config file. - $content | Out-File -FilePath "config.json" - - # Run the stress tests for each .NET runtime. - - ${{ each runtime in parameters.netTestRuntimes }}: - - task: DotNetCoreCLI@2 - displayName: Test [${{runtime}}] - inputs: - command: custom - custom: run - projects: ${{ parameters.testProject }} - arguments: ${{ parameters.buildArguments }} --no-build -f ${{runtime}} -e STRESS_CONFIG_FILE=config.json -- $(testArguments) - - # Run the stress tests for each .NET Framework runtime. - - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: - - task: DotNetCoreCLI@2 - displayName: Test [${{runtime}}] - inputs: - command: custom - custom: run - projects: ${{ parameters.testProject }} - arguments: ${{ parameters.buildArguments }} --no-build -f ${{runtime}} -e STRESS_CONFIG_FILE=config.json -- $(testArguments) diff --git a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml index 9d205b5108..c706ca0c57 100644 --- a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml @@ -27,7 +27,7 @@ parameters: # The prefix to prepend to the job's display name: # - # [] Run Stress Tests + # [] Test Abstractions Package # - name: displayNamePrefix type: string @@ -58,11 +58,15 @@ parameters: default: [] # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string # The pool VM image to use. - - name: vmImage + - name: poolImage type: string jobs: @@ -74,11 +78,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -91,15 +95,13 @@ jobs: # dotnet CLI arguments for build/test/pack commands - name: buildArguments value: >- - --configuration ${{ parameters.buildConfiguration }} + -p:Configuration=${{ parameters.buildConfiguration }} --verbosity ${{ parameters.dotnetVerbosity }} # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public project. # This is defined with a non-standard Platform of 'AnyCPU', and will fail - # the builds if left defined. The stress tests solution does not require - # any specific Platform, and so its solution file doesn't support any - # non-standard platforms. + # the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. @@ -124,7 +126,7 @@ jobs: displayName: '[Debug] Print Environment Variables' # Install the .NET SDK and Runtimes. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} runtimes: [8.x, 9.x] diff --git a/eng/pipelines/jobs/test-azure-package-ci-job.yml b/eng/pipelines/jobs/test-azure-package-ci-job.yml index 53ca26366f..e927eb6e6e 100644 --- a/eng/pipelines/jobs/test-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/test-azure-package-ci-job.yml @@ -26,10 +26,9 @@ parameters: type: string default: Logging.Artifacts - # The Abstractions package verion to depend on. - # - # This is used when the referenceType is 'Package'. - - name: abstractionsPackageVersion + # The version to apply to the SqlClient family packages (SqlClient, Abstractions, Logging). They + # all share this version. Used when referenceType is 'Package'. + - name: packageVersion type: string # The type of build to test (Release or Debug) @@ -46,7 +45,7 @@ parameters: # The prefix to prepend to the job's display name: # - # [] Run Stress Tests + # [] Test Azure Package # - name: displayNamePrefix type: string @@ -73,11 +72,20 @@ parameters: type: string default: MDS.Artifacts - # The MDS package verion to depend on. + # The name of the SqlServer pipeline artifacts to download. + # + # This is used when the referenceType is 'Package'. MDS depends on + # SqlServer.Server, so the package must be available for transitive restore. + - name: sqlServerArtifactsName + type: string + default: SqlServer.Artifacts + + # The SqlServer package version to depend on. # # This is used when the referenceType is 'Package'. - - name: mdsPackageVersion + - name: sqlServerPackageVersion type: string + default: '' # The list of .NET Framework runtimes to test against. - name: netFrameworkRuntimes @@ -90,6 +98,10 @@ parameters: default: [] # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string @@ -113,14 +125,8 @@ parameters: type: stepList default: [] - # True if the VM image includes a local SQL Server that supports connections - # via integrated security. - - name: supportsIntegratedSecurity - type: boolean - default: false - # The pool VM image to use. - - name: vmImage + - name: poolImage type: string jobs: @@ -132,11 +138,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -149,18 +155,16 @@ jobs: # dotnet CLI arguments for build/test/pack commands. - name: buildArguments value: >- - --configuration ${{ parameters.buildConfiguration }} + -p:Configuration=${{ parameters.buildConfiguration }} --verbosity ${{ parameters.dotnetVerbosity }} -p:ReferenceType=${{ parameters.referenceType }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} + -p:SqlClientPackageVersion=${{ parameters.packageVersion }} + -p:SqlServerPackageVersion=${{ parameters.sqlServerPackageVersion }} # Explicitly unset the $PLATFORM environment variable that is set by the # 'ADO Build properties' Library in the ADO SqlClientDrivers public # project. This is defined with a non-standard Platform of 'AnyCPU', and - # will fail the builds if left defined. The stress tests solution does - # not require any specific Platform, and so its solution file doesn't - # support any non-standard platforms. + # will fail the builds if left defined. # # Note that Azure Pipelines will inject this variable as PLATFORM into the # environment of all tasks in this job. @@ -210,8 +214,18 @@ jobs: artifactName: ${{ parameters.mdsArtifactsName }} targetPath: $(Build.SourcesDirectory)/packages + # Download the SqlServer package artifacts into packages/. + # + # MDS depends on SqlServer.Server, so the package must be available for + # transitive restore. + - task: DownloadPipelineArtifact@2 + displayName: Download SqlServer Package Artifacts + inputs: + artifactName: ${{ parameters.sqlServerArtifactsName }} + targetPath: $(Build.SourcesDirectory)/packages + # Install the .NET SDK and Runtimes. - - template: /eng/pipelines/steps/install-dotnet.yml@self + - template: /eng/pipelines/common/steps/install-dotnet.yml@self parameters: debug: ${{ parameters.debug }} runtimes: [8.x, 9.x] @@ -229,15 +243,14 @@ jobs: debug: ${{ parameters.debug }} saPassword: ${{ parameters.saPassword }} - # The config.json file has many options, but only some of them are + # The config.jsonc file has many options, but only some of them are # used by the Azure package tests. We only specify the ones that are # necessary here. AADServicePrincipalId: $(AADServicePrincipalId) AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) # macOS doesn't support managed identities. - ManagedIdentitySupported: ${{ not(eq(parameters.vmImage, 'macos-latest')) }} - SupportsIntegratedSecurity: ${{ parameters.supportsIntegratedSecurity }} + ManagedIdentitySupported: ${{ not(eq(parameters.poolImage, 'macos-latest')) }} TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) UserManagedIdentityClientId: $(UserManagedIdentityClientId) WorkloadIdentityFederationServiceConnectionId: $(WorkloadIdentityFederationServiceConnectionId) diff --git a/eng/pipelines/libraries/ci-build-variables.yml b/eng/pipelines/libraries/ci-build-variables.yml index 2779277678..774be86113 100644 --- a/eng/pipelines/libraries/ci-build-variables.yml +++ b/eng/pipelines/libraries/ci-build-variables.yml @@ -4,64 +4,15 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This file is only included in PR and CI pipelines for the Abstractions, AKV, -# Azure, Logging, and MDS projects. +# Variables for PR and CI pipelines. Package versions are computed by each project's +# Versions.props using BuildNumber + buildSuffix. No per-package version variables are needed. +# The buildSuffix itself is set by each pipeline via the core template parameter. variables: + - group: sqlclient-pipeline-config-v1 - group: ADO Build properties - group: ADO Test Configuration Properties - # C# assembly versions must be in the format: Major.Minor.Build.Revision, but - # $(Build.BuildNumber) has the format XXX.YY. Additionally, each version part - # must be a positive 16-bit integer less than 65535. Simply concatenating - # both parts of $(Build.BuildNumber) could produce values larger than 65534, - # so we must omit the second part entirely. Unfortunately, this may result - # in multiple subsequent pipline builds using the same C# assembly versions. - # The package versions will not be affected and will show the complete - # $(Build.BuildNumber) values. - - name: assemblyBuildNumber - value: $[ split(variables['Build.BuildNumber'], '.')[0] ] - - # Abstractions library assembly file version - - name: abstractionsAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # Abstractions library NuGet package version - - name: abstractionsPackageVersion - value: 1.0.0.$(Build.BuildNumber)-ci - - # AKV provider assembly file version - - name: akvAssemblyFileVersion - value: 7.0.0.$(assemblyBuildNumber) - - # AKV provider NuGet package version - - name: akvPackageVersion - value: 7.0.0.$(Build.BuildNumber)-ci - - # Azure library assembly file version - - name: azureAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # Azure library NuGet package version - - name: azurePackageVersion - value: 1.0.0.$(Build.BuildNumber)-ci - - # Logging library assembly file version - - name: loggingAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # Logging library NuGet package version - - name: loggingPackageVersion - value: 1.0.0.$(Build.BuildNumber)-ci - - # MDS library assembly file version - - name: mdsAssemblyFileVersion - value: 7.0.0.$(assemblyBuildNumber) - - # MDS library NuGet package version - - name: mdsPackageVersion - value: 7.0.0.$(Build.BuildNumber)-ci - # Local NuGet feed directory where downloaded pipeline artifacts are placed. # NuGet.config references this as a local package source for restore. - name: localFeedPath diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml new file mode 100644 index 0000000000..6576627a6a --- /dev/null +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -0,0 +1,264 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Generic job template for building and signing simple build.proj packages. +# +# The job builds DLLs via build.proj, ESRP signs the DLLs, creates NuGet packages from the signed +# DLLs, ESRP signs the NuGet packages, and finally copies all the build output to the job output. + +parameters: + # Common Build Job Properties ############################################ + - name: apiScanDllPath + type: string + + - name: apiScanPdbPath + type: string + + # The APIScan registration version for the package being built. This is the major.minor value + # derived from the canonical package version by the compute_versions stage. The package's + # name/version pair must already be registered with APIScan before the build runs. + - name: apiScanSoftwareVersion + type: string + + # True to enable ESRP malware scanning and code signing steps, which should not be run on + # non-official pipelines as they access production resources. If true, Signing* parameters must + # be provided. + - name: shouldSignPackage + type: boolean + + # Signing Parameters ----------------------------------------------------- + # @TODO: Signing Parameters Object + + - name: signingAppRegistrationClientId + type: string + + - name: signingAppRegistrationTenantId + type: string + + - name: signingAuthAkvName + type: string + + - name: signingAuthSignCertName + type: string + + - name: signingEsrpClientId + type: string + + - name: signingEsrpConnectedServiceName + type: string + + # Package Properties ##################################################### + + # A list of packages that the package being built with this job depends on. Each entry in the + # object should have the fields: + # * artifactName - Name of the artifact published in a previous job that contains the desired + # version of the dependency. Set to an empty string to reference a package that is NOT built in + # this run (e.g. a previously published SqlServer package): the version argument is still + # applied, but no artifact is downloaded, so the dependency is restored from NuGet instead of + # the local feed. + # * shortName - Short name of the dependency. This name should correspond to the short name used + # when building the package via build.proj + # * version - Pre-computed version of the dependency (from compute-versions stage). + - name: dependencies + type: object + default: [] + + # Assembly file version to stamp (required). Pre-computed by the compute-versions stage so no + # build job re-derives it. The assembly version is derived from this by Versions.props. + - name: fileVersion + type: string + + # The full name of the package. This is used in the job name, and to form DLL and PDB filenames + # for APIScan. + - name: packageFullName + type: string + values: + - 'Microsoft.Data.SqlClient' + - 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' + - 'Microsoft.Data.SqlClient.Extensions.Abstractions' + - 'Microsoft.Data.SqlClient.Extensions.Azure' + - 'Microsoft.Data.SqlClient.Internal.Logging' + - 'Microsoft.SqlServer.Server' + + # Short package name used in the job name, display strings, filesystem paths, and as a suffix for + # the default Build and Pack targets if those aren't specified. + - name: packageShortName + type: string + values: + - 'Abstractions' + - 'AkvProvider' + - 'Azure' + - 'Logging' + - 'SqlClient' + - 'SqlServer' + + # Suffix appended to PackageVersion to select the canonical build.proj version property. + - name: versionPropertySuffix + type: string + values: + - SqlClient + - SqlServer + + # Package version (required). Pre-computed by compute-versions stage. + - name: packageVersion + type: string + +jobs: + - job: 'build_package_${{ parameters.packageShortName }}' + displayName: 'Build: ${{ parameters.packageFullName }}' + pool: + type: windows + + variables: + # Placeholder for the Microsoft.SqlServer.Server package-version argument (empty unless this + # package depends on SqlServer). Constructed below from parameters.dependencies. + sqlServerVersionArgument: '' + + ob_outputDirectory: '$(JOB_OUTPUT)' + + # APIScan per-job configuration. This job template is the single place where the APIScan + # software name and version are set; the pipelines' globalSdl blocks deliberately leave them + # unset so that every scan is attributed to the package it actually covers. + ob_sdl_apiscan_softwareFolder: ${{ parameters.apiScanDllPath }} + ob_sdl_apiscan_symbolsFolder: ${{ parameters.apiScanPdbPath }} + ob_sdl_apiscan_softwareName: ${{ parameters.packageFullName }} + ob_sdl_apiscan_versionNumber: ${{ parameters.apiScanSoftwareVersion }} + + # SBOM identity for this job's artifact. OneBranch reads the SBOM package name/version only + # from the pipeline's globalSdl block, which has no per-job form, so that block indirects + # through these variables and each build job supplies its own values. Jobs that publish no + # packages set ob_sdl_sbom_enabled to false instead. + sbomPackageName: ${{ parameters.packageFullName }} + sbomPackageVersion: ${{ parameters.packageVersion }} + + steps: + - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self + + # Localized resources ship with the SqlClient driver. Validate them before analysis and + # building so missing or untranslated strings fail every SqlClient build. + - ${{ if eq(parameters.packageShortName, 'SqlClient') }}: + - template: /eng/pipelines/onebranch/steps/validate-localization-step.yml@self + + - ${{ each package in parameters.dependencies }}: + # Build the dependency version arguments passed to the build/pack/analysis steps. The + # SqlClient family shares a single version via Central Package Management, and those steps + # already pass -p:PackageVersionSqlClient (the package's own version), which pins both the + # package being built and every family dependency reference. Family dependencies + # therefore need no extra version argument here -- only a Microsoft.SqlServer.Server + # dependency, which is versioned separately, must be pinned explicitly via + # -p:PackageVersionSqlServer (this applies whether SqlServer is freshly built this run or + # restored from its published NuGet package). + - ${{ if eq(package.shortName, 'SqlServer') }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=sqlServerVersionArgument]$(sqlServerVersionArgument) -p:PackageVersionSqlServer=${{ package.version }}" + displayName: "Append SqlServer Version Argument - ${{ package.shortName }}" + + # Download the pipeline artifact for this dependency package when it was built in this + # run. An explicitly empty artifactName restores the dependency from NuGet instead. + - ${{ if ne(package.artifactName, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download Artifact - ${{ package.shortName }}" + inputs: + artifactName: ${{ package.artifactName }} + targetPath: $(REPO_ROOT)/packages + + # Install the .NET SDK. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + # Restore dotnet local tools (pwsh, apicompat, etc.). Required by build.proj targets + # such as _CheckPwshToolRestored that run during RoslynAnalyzers and Build. + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + # Run Roslyn analysis. This step is self-contained: it performs its own build into an + # isolated output location, so it can run at any point in the job without clobbering the + # real build output below. + - template: /eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml@self + parameters: + dependencyArguments: $(sqlServerVersionArgument) + packageShortName: ${{ parameters.packageShortName }} + fileVersion: ${{ parameters.fileVersion }} + versionPropertySuffix: ${{ parameters.versionPropertySuffix }} + packageVersion: ${{ parameters.packageVersion }} + + # Build the package, producing DLLs only (no NuGet package yet). + - template: /eng/pipelines/onebranch/steps/build-buildproj-step.yml@self + parameters: + buildConfiguration: Release + dependencyArguments: $(sqlServerVersionArgument) + packageShortName: ${{ parameters.packageShortName }} + fileVersion: ${{ parameters.fileVersion }} + versionPropertySuffix: ${{ parameters.versionPropertySuffix }} + packageVersion: ${{ parameters.packageVersion }} + + - ${{ if eq(parameters.shouldSignPackage, true) }}: + # ESRP sign the DLLs. + - template: /eng/pipelines/onebranch/steps/esrp-dll-signing-step.yml@self + parameters: + appRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + appRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + authAkvName: '${{ parameters.signingAuthAkvName }}' + authSignCertName: '${{ parameters.signingAuthSignCertName }}' + esrpClientId: '${{ parameters.signingEsrpClientId }}' + esrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + pattern: | + **/${{ parameters.packageFullName }}.dll + **/${{ parameters.packageFullName }}.resources.dll + + # Copy signed/unsigned DLLs and PDBs to APIScan folders. + - task: CopyFiles@2 + displayName: Copy DLLs for APIScan + inputs: + SourceFolder: $(BUILD_OUTPUT)/${{ parameters.packageFullName }} + Contents: "**/${{ parameters.packageFullName }}.dll" + TargetFolder: '${{ parameters.apiScanDllPath }}' + # We must preserve the folder structure since our C# projects may produce multiple + # identically named DLLs for different target frameworks (e.g. netstandard2.0, net5.0, + # etc.), and we need to keep those separate for APIScan to work correctly. + flattenFolders: false + + - task: CopyFiles@2 + displayName: Copy PDBs for APIScan + inputs: + SourceFolder: $(BUILD_OUTPUT)/${{ parameters.packageFullName }} + Contents: "**/${{ parameters.packageFullName }}.pdb" + TargetFolder: '${{ parameters.apiScanPdbPath }}' + # We must preserve the folder structure since our C# projects may produce multiple + # identically named DLLs for different target frameworks (e.g. netstandard2.0, net5.0, + # etc.), and we need to keep those separate for APIScan to work correctly. + flattenFolders: false + + # Pack the signed DLLs into NuGet package (NoBuild=true). + - template: /eng/pipelines/onebranch/steps/pack-buildproj-step.yml@self + parameters: + buildConfiguration: Release + dependencyArguments: $(sqlServerVersionArgument) + packageFullName: ${{ parameters.packageFullName }} + packageShortName: ${{ parameters.packageShortName }} + fileVersion: ${{ parameters.fileVersion }} + versionPropertySuffix: ${{ parameters.versionPropertySuffix }} + packageVersion: ${{ parameters.packageVersion }} + + - ${{ if eq(parameters.shouldSignPackage, true) }}: + # ESRP sign the NuGet package. + - template: /eng/pipelines/onebranch/steps/esrp-nuget-signing-step.yml@self + parameters: + appRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + appRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + authAkvName: '${{ parameters.signingAuthAkvName }}' + authSignCertName: '${{ parameters.signingAuthSignCertName }}' + esrpClientId: '${{ parameters.signingEsrpClientId }}' + esrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + searchPath: '$(BUILD_OUTPUT)' + searchPattern: '**/${{ parameters.packageFullName}}.*nupkg' + + # Copy the contents of the build output for this package to the artifacts folder + - task: CopyFiles@2 + displayName: Copy Build Output to Artifacts Folder + inputs: + SourceFolder: '$(BUILD_OUTPUT)/${{ parameters.packageFullName }}' + Contents: '**/*.*' + TargetFolder: '$(JOB_OUTPUT)' + flattenFolders: false diff --git a/eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml b/eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml deleted file mode 100644 index 809690a886..0000000000 --- a/eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml +++ /dev/null @@ -1,200 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# Generic job template for building and signing simple csproj-based packages, currently: -# -# - Abstractions -# - Azure -# - Logging -# - SqlServer -# -# The job builds DLLs via build.proj, ESRP signs the DLLs, then creates NuGet packages from the -# signed DLLs, and finally ESRP signs the NuGet packages. - -parameters: - # Short package name used in the job name, display strings, filesystem paths, and as a suffix for - # the default Build and Pack targets if those aren't specified. - - name: packageName - type: string - values: - - Abstractions - - AkvProvider - - Azure - - Logging - - SqlServer - - # The full name of the package. This is used in the job name, and to form DLL and PDB filenames - # for APIScan. - - name: packageFullName - type: string - - # The version of the package. This is used for symbol publishing. It is not used for the DLL or - # NuGet package versions since those are supplied via the versionProperties parameter. - - name: packageVersion - type: string - - # The MSBuild build target in build.proj (e.g. BuildLogging). If not specified, defaults to - # Build. - - name: buildTarget - type: string - default: "" - - # The MSBuild pack target in build.proj (e.g. PackLogging). If not specified, defaults to - # Pack. - - name: packTarget - type: string - default: "" - - # The C# build configuration to build (e.g. Debug or Release). - - name: buildConfiguration - type: string - values: - - Debug - - Release - default: Release - - # Additional MSBuild -p: arguments for version properties. These may include versions of - # packages this package depends on, or versions for this package itself. - - name: versionProperties - type: string - default: "" - - # Assembly file version for APIScan (e.g. 1.0.0.12345). - - name: assemblyFileVersion - type: string - - # True to publish symbols to private and public servers. - - name: publishSymbols - type: boolean - - # Values required by ESRP tasks. - - name: esrpConnectedServiceName - type: string - - - name: esrpClientId - type: string - - - name: appRegistrationClientId - type: string - - - name: appRegistrationTenantId - type: string - - - name: authAkvName - type: string - - - name: authSignCertName - type: string - - # Optional list of pipeline artifacts to download before building. Each entry is an object - # with 'artifactName' (the pipeline artifact name) and 'displayName' (used in the task label). - # This replaces hard-coded packageName conditionals so callers declare their own dependencies. - - name: downloadArtifacts - type: object - default: [] - -jobs: - - job: build_package_${{ parameters.packageName }} - displayName: Build ${{ parameters.packageFullName }} - pool: - type: windows - - variables: - ob_outputDirectory: $(PACK_OUTPUT) - # APIScan configuration for this Extension package - ob_sdl_apiscan_enabled: true - ob_sdl_apiscan_softwareFolder: $(Build.SourcesDirectory)/apiScan/${{ parameters.packageName }}/dlls - ob_sdl_apiscan_symbolsFolder: $(Build.SourcesDirectory)/apiScan/${{ parameters.packageName }}/pdbs - ob_sdl_apiscan_softwarename: ${{ parameters.packageFullName }} - ob_sdl_apiscan_versionNumber: ${{ parameters.assemblyFileVersion }} - - buildTarget: ${{ coalesce(parameters.buildTarget, format('Build{0}', parameters.packageName)) }} - packTarget: ${{ coalesce(parameters.packTarget, format('Pack{0}', parameters.packageName)) }} - - steps: - - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self - - # Download any pipeline artifacts this package depends on. - - ${{ each artifact in parameters.downloadArtifacts }}: - - task: DownloadPipelineArtifact@2 - displayName: Download ${{ artifact.displayName }} - inputs: - artifactName: ${{ artifact.artifactName }} - targetPath: $(Build.SourcesDirectory)/packages - - # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self - - # Perform Roslyn analysis before building, since this step will clobber build output. - - template: /eng/pipelines/onebranch/steps/code-analyze-step.yml@self - parameters: - msBuildArguments: >- - -t:$(buildTarget) - -p:Configuration=${{ parameters.buildConfiguration }} - -p:ReferenceType=Package - ${{ parameters.versionProperties }} - - # Build the package, producing DLLs only (no NuGet package yet). - - template: /eng/pipelines/onebranch/steps/compound-build-csproj-step.yml@self - parameters: - buildTarget: $(buildTarget) - buildConfiguration: ${{ parameters.buildConfiguration }} - versionProperties: ${{ parameters.versionProperties }} - - # ESRP sign the DLLs. - - template: /eng/pipelines/onebranch/steps/compound-esrp-dll-signing-step.yml@self - parameters: - appRegistrationClientId: ${{ parameters.appRegistrationClientId }} - appRegistrationTenantId: ${{ parameters.appRegistrationTenantId }} - authAkvName: ${{ parameters.authAkvName }} - authSignCertName: ${{ parameters.authSignCertName }} - esrpClientId: ${{ parameters.esrpClientId }} - esrpConnectedServiceName: ${{ parameters.esrpConnectedServiceName }} - pattern: ${{ parameters.packageFullName }}.dll - - # Copy signed DLLs and PDBs to APIScan folders. - - task: CopyFiles@2 - displayName: Copy DLLs for APIScan - inputs: - SourceFolder: $(BUILD_OUTPUT)/Package/bin - Contents: "**/${{ parameters.packageFullName }}.dll" - TargetFolder: $(ob_sdl_apiscan_softwareFolder) - # We must preserve the folder structure since our C# projects may produce multiple - # identically named DLLs for different target frameworks (e.g. netstandard2.0, net5.0, - # etc.), and we need to keep those separate for APIScan to work correctly. - flattenFolders: false - - - task: CopyFiles@2 - displayName: Copy PDBs for APIScan - inputs: - SourceFolder: $(BUILD_OUTPUT)/Package/bin - Contents: "**/${{ parameters.packageFullName }}.pdb" - TargetFolder: $(ob_sdl_apiscan_symbolsFolder) - flattenFolders: false - - # Pack the signed DLLs into NuGet package (NoBuild=true). - - template: /eng/pipelines/onebranch/steps/compound-pack-csproj-step.yml@self - parameters: - packTarget: $(packTarget) - buildConfiguration: ${{ parameters.buildConfiguration }} - versionProperties: ${{ parameters.versionProperties }} - - # ESRP sign the NuGet package. - - template: /eng/pipelines/onebranch/steps/compound-esrp-nuget-signing-step.yml@self - parameters: - appRegistrationClientId: ${{ parameters.appRegistrationClientId }} - appRegistrationTenantId: ${{ parameters.appRegistrationTenantId }} - authAkvName: ${{ parameters.authAkvName }} - authSignCertName: ${{ parameters.authSignCertName }} - esrpClientId: ${{ parameters.esrpClientId }} - esrpConnectedServiceName: ${{ parameters.esrpConnectedServiceName }} - - # Publish symbols to servers - - ${{ if eq(parameters.publishSymbols, true) }}: - - template: /eng/pipelines/onebranch/steps/publish-symbols-step.yml@self - parameters: - packageFullName: ${{ parameters.packageFullName }} - packageVersion: ${{ parameters.packageVersion }} diff --git a/eng/pipelines/onebranch/jobs/build-signed-sqlclient-package-job.yml b/eng/pipelines/onebranch/jobs/build-signed-sqlclient-package-job.yml deleted file mode 100644 index a422a5351c..0000000000 --- a/eng/pipelines/onebranch/jobs/build-signed-sqlclient-package-job.yml +++ /dev/null @@ -1,137 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# This file is only included in MDS OneBranch Official pipelines. - -parameters: - # True to publish symbols to public and private feeds after the build completes. - - name: publishSymbols - type: boolean - - # True if this is a preview build, which uses the preview version numbers from - # common-variables.yml. - - name: isPreview - type: boolean - -jobs: -- job: build_package_SqlClient - displayName: 'Build Microsoft.Data.SqlClient' - pool: - type: windows # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs - - variables: - ob_outputDirectory: $(PACK_OUTPUT) - # APIScan configuration for this Extension package - ob_sdl_apiscan_enabled: true - ob_sdl_apiscan_softwareFolder: $(Build.SourcesDirectory)/apiScan/SqlClient/dlls - ob_sdl_apiscan_symbolsFolder: $(Build.SourcesDirectory)/apiScan/SqlClient/pdbs - ob_sdl_apiscan_softwarename: Microsoft.Data.SqlClient - ob_sdl_apiscan_versionNumber: $(assemblyBuildNumber) - - ${{ if parameters.isPreview }}: - abstractionsPackageVersion: $(abstractionsPackagePreviewVersion) - loggingPackageVersion: $(loggingPackagePreviewVersion) - mdsPackageVersion: $(mdsPackagePreviewVersion) - - steps: - - script: SET - displayName: 'Print Environment Variables' - - # Download the Abstractions and Logging packages from the previous stage into - # packages/ so that they're available via the local NuGet feed when restoring MDS. - # MDS depends on both Extensions.Abstractions and Internal.Logging. - - task: DownloadPipelineArtifact@2 - displayName: Download Abstractions Package - inputs: - artifactName: $(abstractionsArtifactsName) - targetPath: $(Build.SourcesDirectory)/packages - - - task: DownloadPipelineArtifact@2 - displayName: Download Logging Package - inputs: - artifactName: $(loggingArtifactsName) - targetPath: $(Build.SourcesDirectory)/packages - - # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self - - # Build our tooling, which is required by the analysis step below, but - # shouldn't be analyzed itself. - - task: MSBuild@1 - displayName: 'Build Tooling' - inputs: - solution: '**/build.proj' - configuration: Release - msbuildArguments: -t:BuildTools - - # Perform analysis before building, since this step will clobber build output. - - template: /eng/pipelines/onebranch/steps/code-analyze-step.yml@self - - # Build MDS, producing signed DLLs. - - template: /eng/pipelines/onebranch/steps/build-all-configurations-signed-dlls-step.yml@self - parameters: - # These variables are sourced from common-variables.yml. - abstractionsAssemblyFileVersion: $(abstractionsAssemblyFileVersion) - abstractionsPackageVersion: $(abstractionsPackageVersion) - loggingAssemblyFileVersion: $(loggingAssemblyFileVersion) - loggingPackageVersion: $(loggingPackageVersion) - mdsAssemblyFileVersion: $(mdsAssemblyFileVersion) - mdsPackageVersion: $(mdsPackageVersion) - - - template: /eng/pipelines/onebranch/steps/esrp-code-signing-step.yml@self - parameters: - artifactType: dll - sourceRoot: $(BUILD_OUTPUT) - dllPattern: 'Microsoft.Data.SqlClient.dll' - - - template: /eng/pipelines/onebranch/steps/esrp-code-signing-step.yml@self - parameters: - artifactType: dll - sourceRoot: $(BUILD_OUTPUT) - dllPattern: 'Microsoft.Data.SqlClient.resources.dll' - - - template: /eng/pipelines/common/templates/steps/generate-nuget-package-step.yml@self - parameters: - buildConfiguration: Release - displayName: 'Create MDS NuGet Package' - generateSymbolsPackage: true - installNuget: false - nuspecPath: $(nuspecPath) - outputDirectory: $(PACK_OUTPUT) - packageVersion: $(mdsPackageVersion) - properties: 'AbstractionsPackageVersion=$(abstractionsPackageVersion);LoggingPackageVersion=$(loggingPackageVersion)' - referenceType: Package - - - template: /eng/pipelines/onebranch/steps/esrp-code-signing-step.yml@self - parameters: - artifactType: pkg - - # Copy signed DLLs and PDBs to APIScan folders. - - task: CopyFiles@2 - displayName: Copy DLLs for APIScan - inputs: - SourceFolder: $(BUILD_OUTPUT)/Package/bin - Contents: '**/Microsoft.Data.SqlClient.dll' - TargetFolder: $(ob_sdl_apiscan_softwareFolder) - # We must preserve the folder structure since our C# projects may produce multiple - # identically named DLLs for different target frameworks (e.g. netstandard2.0, net5.0, - # etc.), and we need to keep those separate for APIScan to work correctly. - flattenFolders: false - - - task: CopyFiles@2 - displayName: Copy PDBs for APIScan - inputs: - SourceFolder: $(BUILD_OUTPUT)/Package/bin - Contents: '**/Microsoft.Data.SqlClient.pdb' - TargetFolder: $(ob_sdl_apiscan_symbolsFolder) - flattenFolders: false - - # Publish symbols to servers - - ${{ if eq(parameters.publishSymbols, true) }}: - - template: /eng/pipelines/onebranch/steps/publish-symbols-step.yml@self - parameters: - packageFullName: Microsoft.Data.SqlClient - packageVersion: $(mdsPackageVersion) diff --git a/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml b/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml index c04a2eceef..e63c72bf58 100644 --- a/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml +++ b/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml @@ -38,15 +38,22 @@ parameters: # artifact. For example, if we're publishing the SqlClient package, and the build job publishes a # pipeline artifact with the following structure: # - # drop_BuildAndTest_PackageReference/ - # ├── Microsoft.Data.SqlClient.5.0.0.nupkg - # ├── Microsoft.Data.SqlClient.5.0.0.snupkg - # ├── Microsoft.Data.SqlClient.Extensions.Abstractions.1.0.0.nupkg - # └── other-file.txt + # drop_build_independent_build_logging/ + # ├── Package-Release/ + # │ ├── Microsoft.Data.SqlClient.5.0.0.nupkg + # │ └── Microsoft.Data.SqlClient.5.0.0.snupkg + # └── symbols/ + # └── ... # - # Then the packagePath should be 'Microsoft.Data.SqlClient.5.0.0.nupkg'. + # Then the packagePath should be 'Package-Release/Microsoft.Data.SqlClient.5.0.0.nupkg'. # - # Defaults to '${{ parameters.packageName }}.*.nupkg' to match any version of the package. + # When left empty, it defaults to '*/${{ parameters.packageName }}.*.nupkg' to match any version of + # the package (see the packageToPush variable; ADO's coalesce() skips the empty default and falls + # back to that glob). The leading '*/' matches whatever single configuration folder the build + # produced: the SqlClient package is packed into 'Package-Release/' (PackSqlClient sets + # PackageOutputPath), while the other packages are packed by 'dotnet pack' into a plain 'Release/' + # folder. Only .nupkg files are pushed; .snupkg files are included automatically by NuGet when + # they exist alongside the .nupkg in the same directory. # - name: packagePath type: string @@ -65,13 +72,18 @@ jobs: variables: - name: ob_outputDirectory - value: $(Build.SourcesDirectory)/output + value: $(JOB_OUTPUT) + + # This job republishes an already-built package, whose SBOM came from its build job. It sets + # no sbomPackage* values, so leaving SBOM enabled would emit one with unresolved macros. + - name: ob_sdl_sbom_enabled + value: false - name: artifactPath value: $(Pipeline.Workspace)/${{ parameters.artifactName }} - name: packageToPush - value: $(artifactPath)/${{ coalesce(parameters.packagePath, format('{0}.*.nupkg', parameters.packageName)) }} + value: $(artifactPath)/${{ coalesce(parameters.packagePath, format('*/{0}.*.nupkg', parameters.packageName)) }} # Template context inputs are used to pass parameters to the deployment job since it doesn't # automatically download pipeline artifacts. diff --git a/eng/pipelines/onebranch/jobs/publish-symbols-job.yml b/eng/pipelines/onebranch/jobs/publish-symbols-job.yml new file mode 100644 index 0000000000..ab45b5f4c8 --- /dev/null +++ b/eng/pipelines/onebranch/jobs/publish-symbols-job.yml @@ -0,0 +1,110 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Reusable job template for publishing symbols for a single package. Downloads the pipeline +# artifact produced by a build job, then invokes publish-symbols-step.yml to upload and publish +# the PDB files to the internal and public symbol servers. +# +# Each package's PDBs are published separately to maintain unique naming and versioning. +# PDBs are expected to be located under the 'symbols/' directory within the downloaded +# artifact. + +parameters: + # The pipeline artifact name to download (OneBranch naming: drop__). + - name: artifactName + type: string + + # The full NuGet package name (e.g. Microsoft.Data.SqlClient.Internal.Logging). + - name: packageFullName + type: string + + # Short package name used in the job name (e.g. Logging, SqlServer, SqlClient). + - name: packageShortName + type: string + + # The package version, used for symbol versioning (required, pre-computed by compute-versions stage). + - name: packageVersion + type: string + + # Symbols Publishing Parameters ------------------------------------------ + + - name: symbolsAzureSubscription + type: string + + - name: symbolsPublishProjectName + type: string + + - name: symbolsPublishServer + type: string + + - name: symbolsPublishTokenUri + type: string + + - name: symbolsUploadAccount + type: string + +jobs: + - job: 'publish_symbols_${{ parameters.packageShortName }}' + displayName: 'Publish Symbols: ${{ parameters.packageFullName }}' + pool: + type: linux + + variables: + # OneBranch requires ob_outputDirectory to be set. Pipeline Artifacts are always on and + # cannot be disabled. To prevent this job from publishing artifacts, a .artifactignore + # that excludes all files is written into ob_outputDirectory before the auto-publish step. + - name: ob_outputDirectory + value: $(Build.SourcesDirectory)/no_publish + # Disable SDL scanning — this job only uploads/publishes PDBs and produces no + # assemblies to scan. APIScan and BinSkim are handled by the build jobs. + - name: ob_sdl_apiscan_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + # No packages are published here, so there is nothing to describe in an SBOM. + - name: ob_sdl_sbom_enabled + value: false + # Path where the downloaded artifact will be placed. + - name: artifactPath + value: '$(Pipeline.Workspace)/${{ parameters.packageFullName }}' + # PublishSymbols@2 runs on the OneBranch host agent (outside the build container) due to 1ES + # Pipeline Template credential isolation. On Linux, the host resolves to the Microsoft org by + # default. Setting this variable at job level ensures the task sees it and connects to the + # correct org's symbol store. + # + # Reference: + # https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL#Option_B:_OneBranch + - name: ArtifactServices.Symbol.AccountName + value: ${{ parameters.symbolsUploadAccount }} + + steps: + # Create ob_outputDirectory with a .artifactignore that excludes everything, + # so OneBranch's auto-publish uploads an empty artifact. + - pwsh: | + New-Item -Path "$(ob_outputDirectory)" -ItemType Directory -Force + "**" | Out-File -FilePath "$(ob_outputDirectory)/.artifactignore" -Encoding ascii + displayName: 'Suppress artifact publishing' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download ${{ parameters.packageFullName }} Artifact' + inputs: + artifactName: '${{ parameters.artifactName }}' + targetPath: '${{ variables.artifactPath }}' + + - template: /eng/pipelines/onebranch/steps/publish-symbols-step.yml@self + parameters: + artifactName: '${{ parameters.packageFullName }}_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_${{ parameters.packageVersion }}_$(System.TimelineId)_$(System.JobAttempt)' + azureSubscription: '${{ parameters.symbolsAzureSubscription }}' + packageName: '${{ parameters.packageFullName }}' + publishProjectName: '${{ parameters.symbolsPublishProjectName }}' + publishServer: '${{ parameters.symbolsPublishServer }}' + publishToInternal: true + publishToPublic: true + publishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + searchPattern: '**/${{ parameters.packageFullName }}.pdb' + symbolsFolder: '${{ variables.artifactPath }}' + uploadAccount: '${{ parameters.symbolsUploadAccount }}' + version: '${{ parameters.packageVersion }}' diff --git a/eng/pipelines/onebranch/jobs/validate-packages-job.yml b/eng/pipelines/onebranch/jobs/validate-packages-job.yml new file mode 100644 index 0000000000..f1501c7d1d --- /dev/null +++ b/eng/pipelines/onebranch/jobs/validate-packages-job.yml @@ -0,0 +1,206 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Validates every NuGet package produced by this run. +# +# All packages are downloaded into a single tree and validated together in one job, rather than one +# job per package, so that PackageValidator can apply its cross-package rules: the SqlClient family +# must share a single version, and their inter-package dependency ranges must agree. +# +# The job runs on Windows because Authenticode verification has no equivalent on the Linux agents. +# PackageValidator itself is cross-platform, so only the signature checks are OS-bound. + +parameters: + # Package Parameters ----------------------------------------------------- + + - name: abstractionsArtifactsName + type: string + + - name: akvProviderArtifactsName + type: string + + - name: azureArtifactsName + type: string + + - name: loggingArtifactsName + type: string + + - name: sqlClientArtifactsName + type: string + + - name: sqlServerArtifactsName + type: string + + # Version Parameters ----------------------------------------------------- + # Pre-computed by the compute-versions stage. Validation asserts the produced packages carry + # exactly these versions, so nothing here is re-derived. + + - name: sqlClientPackageVersion + type: string + + - name: sqlClientFileVersion + type: string + + - name: sqlServerPackageVersion + type: string + + - name: sqlServerFileVersion + type: string + + # Behaviour Parameters --------------------------------------------------- + + # Whether Microsoft.SqlServer.Server was built this run. When false there is no SqlServer + # artifact to download and no SqlServer package in the drop to validate. + - name: buildSqlServer + type: boolean + + # True for official builds, which sign their packages and assemblies. Signature verification is + # skipped otherwise, because non-official runs deliberately produce unsigned output. + - name: isOfficial + type: boolean + +jobs: + - job: validate_packages + displayName: 'Validate Packages' + + pool: + type: windows + + # 1ES auto-injects Roslyn into any job holding a DotNetCoreCLI build task, which here is only + # the PackageValidator tool build -- never shipped, so out of SDL scope. + templateContext: + sdl: + roslyn: + enabled: false + + variables: + - name: ob_outputDirectory + value: '$(JOB_OUTPUT)' + + # This job inspects already-built packages and produces no assemblies, so it has nothing for + # APIScan or BinSkim to scan and no shipping component to describe in an SBOM. The build + # jobs cover all three for the packages they produce. + - name: ob_sdl_apiscan_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_sbom_enabled + value: false + + # Every package artifact is downloaded beneath this root, each into its own subdirectory so + # that identically-named files from different packages cannot collide. + - name: packagesRoot + value: '$(Pipeline.Workspace)/validate-packages' + + - name: extractRoot + value: '$(Pipeline.Workspace)/validate-extract' + + steps: + - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self + + # Only the packages themselves are needed, not the full build output each artifact carries. + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Logging' + inputs: + artifactName: '${{ parameters.loggingArtifactsName }}' + targetPath: '$(packagesRoot)/Logging' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Abstractions' + inputs: + artifactName: '${{ parameters.abstractionsArtifactsName }}' + targetPath: '$(packagesRoot)/Abstractions' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - SqlClient' + inputs: + artifactName: '${{ parameters.sqlClientArtifactsName }}' + targetPath: '$(packagesRoot)/SqlClient' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Azure' + inputs: + artifactName: '${{ parameters.azureArtifactsName }}' + targetPath: '$(packagesRoot)/Azure' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - AkvProvider' + inputs: + artifactName: '${{ parameters.akvProviderArtifactsName }}' + targetPath: '$(packagesRoot)/AkvProvider' + patterns: '**/*.*nupkg' + + - ${{ if eq(parameters.buildSqlServer, true) }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - SqlServer' + inputs: + artifactName: '${{ parameters.sqlServerArtifactsName }}' + targetPath: '$(packagesRoot)/SqlServer' + patterns: '**/*.*nupkg' + + # PackageValidator targets net10.0, which the repo's global.json already pins. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + - template: /eng/pipelines/onebranch/steps/validate-packages-step.yml@self + parameters: + packagesPath: '$(packagesRoot)' + reportPath: '$(JOB_OUTPUT)/validation/package-validation.json' + sqlClientPackageVersion: '${{ parameters.sqlClientPackageVersion }}' + sqlClientFileVersion: '${{ parameters.sqlClientFileVersion }}' + # Omitted when SqlServer is not built: its package is absent from the drop, and the + # validator rejects an expectation with an empty value. + ${{ if eq(parameters.buildSqlServer, true) }}: + sqlServerPackageVersion: '${{ parameters.sqlServerPackageVersion }}' + sqlServerFileVersion: '${{ parameters.sqlServerFileVersion }}' + # The error severity covers only error-severity findings, so every warning/info category + # this job relies on must be named explicitly: missing-symbols, dependency-inconsistency + # and delay-signed are warnings, and unsigned and package-unsigned are info. + # + # Strong-name signing is unconditional in build-buildproj-step.yml, so delay-signed and + # unsigned gate on every run. NuGet package signing is ESRP and runs on official builds + # only, so package-unsigned would fire on every non-official build. + ${{ if eq(parameters.isOfficial, true) }}: + failOn: + - error + - missing-symbols + - dependency-inconsistency + - delay-signed + - unsigned + - package-unsigned + ${{ else }}: + failOn: + - error + - missing-symbols + - dependency-inconsistency + - delay-signed + - unsigned + + # Signature verification, official builds only. PackageValidator reports strong-name and + # NuGet signature *presence* cross-platform; these steps additionally verify that the + # signatures are trusted, which requires the Windows trust store. + - ${{ if eq(parameters.isOfficial, true) }}: + - task: PowerShell@2 + displayName: 'Verify NuGet package signatures' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 + arguments: >- + -PackagesPath "$(packagesRoot)" + + - task: PowerShell@2 + displayName: 'Verify assembly Authenticode signatures' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 + arguments: >- + -PackagesPath "$(packagesRoot)" + -ExtractPath "$(extractRoot)" diff --git a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml deleted file mode 100644 index 656ea8f181..0000000000 --- a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml +++ /dev/null @@ -1,281 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# -parameters: - - # The name of the pipeline artifacts to download prior to building the tests. - - name: artifactName - type: string - - # True if this build is a preview. - - name: isPreview - type: boolean - -jobs: -- job: validate_signed_package - displayName: 'Verify signed package' - - pool: - type: windows # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs - isCustom: true - name: ADO-1ES-Pool - vmImage: 'ADO-MMS22-SQL19' - - variables: # More settings at https://aka.ms/obpipelines/yaml/jobs - - template: /eng/pipelines/onebranch/variables/sqlclient-validation-variables.yml@self - - - name: pathToDownloadedNuget # path to the downloaded nuget files - value: $(Pipeline.Workspace)\${{parameters.artifactName }} - - - ${{ if parameters.isPreview }}: - - name: extractedNugetPath - value: $(extractedNugetRootPath).$(mdsPackagePreviewVersion) - - name: mdsPackageVersion - value: $(mdsPackagePreviewVersion) - - steps: - - script: SET - displayName: 'Print Environment Variables' - - - task: NuGetToolInstaller@1 - displayName: 'Use NuGet' - - - powershell: | - # Displays the paths of all the local cache directories - nuget locals all -List - - #Clears all files from all local cache directories - nuget locals all -Clear - displayName: 'Clear local cache' - - - download: current - artifact: ${{parameters.artifactName}} - patterns: '**/*.*nupkg' - displayName: 'Download NuGet Package' - - - powershell: | - # Install nuget package - Install-Package -Name "Microsoft.Data.SqlClient" -Destination "$(TempFolderName)" -Force -Source $(pathToDownloadedNuget) -SkipDependencies - - Write-Host "--------------------------------------------------" - Write-Host '$(TempFolderName)' - ls $(TempFolderName) - Write-Host "--------------------------------------------------" - displayName: 'Extract Nuget in temp folder' - - - powershell: | - Write-Host "--------------------------------------------------" - Write-Host "This will verify the artifact signature" -ForegroundColor Green - Write-Host "--------------------------------------------------" - - nuget verify -All $(pathToDownloadedNuget)\*.nupkg - nuget verify -All $(pathToDownloadedNuget)\*.snupkg - displayName: 'Verify nuget signature' - - - powershell: | - # Recursively find all .dll files in TempFolder (installed nuget folder) - # Microsoft.Data.SqlClient.dll and Microsoft.Data.SqlClient.resources.dll (in localized folders) should have strong name - $dllFiles = Get-ChildItem -Path $(TempFolderName) -Recurse -Filter *.dll - $badDlls = @() - foreach ($file in $dllFiles) - { - # Run sn.exe to verify the strong name on each dll - $result = & "C:\Program Files (x86)\Microsoft SDKs\Windows\*\bin\NETFX 4.8.1 Tools\sn.exe" -vf $file.FullName - Write-OutPut $result - - # if the dll is not valid, it would be delay signed or test-signed which is not meant for production - if($result[$result.Length-1] -notlike "* is valid") - { - $badDlls += $result[$result.Length-1] - } - } - if($badDlls.Count -gt 0) - { - Write-OutPut "Error: Invalid dlls are detected. Check the list below:" - foreach($dll in $badDlls) - { - Write-Output $dll - } - Exit -1 - } - Write-Host "Strong name has been verified for all dlls" - displayName: 'Verify assembly strong names' - - - powershell: | - # Checks the expected folder names such as lib, ref, runtimes - Get-ChildItem -Path $(extractedNugetPath) -Directory | select Name | foreach { - if('$(expectedFolderNames)'.contains($_.Name)){ - Write-Host expected folder name verfied: $_.Name - } - } - displayName: 'Check expected folder names' - - - powershell: | - # Checks the version of DotNetFramework and DotNet - $countErr = 0 - $countPass = 0 - $excludNamesFromRuntimeFolder = 'lib','win','unix' - - Get-ChildItem -Path $(extractedNugetPath) -Directory | foreach { - $parentname=$_.Name - Write-Host $_.FullName -ForegroundColor yellow - - if($_.Name -ne 'runtimes') { - Get-ChildItem -Path $_.FullName -Directory | select Name | foreach { - if('$(expectedDotnetVersions)'.Contains($_.Name)){ - Write-Host "`tExpected version verified in $parentname": $_.Name -ForegroundColor green - $countPass += 1 - } - else{ - Write-Host "`tUnexpected version detected in $parentname": $_.Name - $countErr += 1 - } - } - } - - elseif ($_.Name -eq 'runtimes'){ - Get-ChildItem -Depth 3 -Path $_.FullName -Exclude $excludNamesFromRuntimeFolder -Directory | foreach{ - if('$(expectedDotnetVersions)'.Contains($_.Name)){ - Write-Host "`tExpected version verfied in $parentname": $_.Name - $countPass += 1 - } - else{ - Write-Host "`tUnexpected version detected": $_.Name -ForegroundColor Red - $countErr += 1 - } - } - } - else{ - Write-Host "`tUnknown folder " $_.Name -ForegroundColor Red - Exit -1 - } - } - - Write-Host "_______________" - Write-Host "Expected: $countPass" - Write-Host "Unexpected: $countErr" - Write-Host "_______________" - if ($countErr -ne 0) - { - Write-Host "Unexpected versions are detected!" -ForegroundColor Red - Exit -1 - } - displayName: 'Check Expected framework' - - - powershell: | - # list all the child items of created temp folder - - #Verify all DLLs unzipped match "expected" hierarchy - - foreach( $folderName in (Get-ChildItem -Path $(extractedNugetPath) -Directory).Name) - { - # List all Childerns of the Path - Get-ChildItem -Path $(extractedNugetPath)\$folderName -Recurse -File - $subFiles = Get-ChildItem -Path $(extractedNugetPath)\$folderName -Recurse -File - - foreach($file in $subFiles) - { - if($subFiles[0].Name -like "*.dll" ) - { - Write-Host $subFiles[0].Name -ForegroundColor Green - Write-Host $subFiles[1].Name -ForegroundColor Green - if(($folderName -eq 'lib') -or ($folderName -eq 'ref')) - { - if($subFiles[2].Name -like "*.dll") - { - Write-Host $subFiles[2].Name -ForegroundColor Green - } - else - { - $subFiles[2].Name - Write-Host "Expected file pattern for localization did not match to *.dll" -ForegroundColor Red - Exit -1 - } - } - } - else - { - $subFiles[0].Name - $subFiles[1].Name - Write-Host "Expected file pattern did not match to *.dll" -ForegroundColor Red - Exit -1 - } - } - } - displayName: 'Verify all DLLs unzipped match "expected" hierarchy' - - powershell: | - # Verify all dlls status are Valid - - $dlls = Get-ChildItem -Path $(extractedNugetPath) -Recurse -Include *.dll - foreach ($status in $dlls | Get-AuthenticodeSignature) - { - if ($status.Status -eq "Valid") - { - Write-Host $status.Status $status.Path - } - else - { - Write-Host "dll status of '$status.Path' is not valid!" -ForegroundColor Red - $status - Exit -1 - } - } - displayName: 'Verify all dlls status are Valid' - - - powershell: | - # This will check each DLL's ProductVersion and FileVersion against - # expected values. - $failed = 0 - - foreach ( $pVersion in Get-ChildItem *.dll -Path $(extractedNugetPath) -Recurse | ForEach-Object versioninfo ) - { - if ($pVersion.ProductVersion -Like '$(mdsPackageVersion)*') - { - Write-Host -ForegroundColor Green "Correct ProductVersion detected for $($pVersion.FileName): $($pVersion.ProductVersion)" - } - else - { - Write-Host -ForegroundColor Red "Wrong ProductVersion detected for $($pVersion.FileName); expected: $(mdsPackageVersion); found: $($pVersion.ProductVersion)" - $failed = 1 - } - - if ($pVersion.FileVersion -eq '$(mdsAssemblyFileVersion)') - { - Write-Host -ForegroundColor Green "Correct FileVersion detected for $($pVersion.FileName): $($pVersion.FileVersion)" - } - else - { - Write-Host -ForegroundColor Red "Wrong FileVersion detected for $($pVersion.FileName); expected $(mdsAssemblyFileVersion); found: $($pVersion.FileVersion)" - $failed = 1 - } - } - - if ($failed -ne 0) - { - Exit -1 - } - - Get-ChildItem *.dll -Path $(extractedNugetPath) -Recurse | ForEach-Object VersionInfo | Format-List - displayName: 'Verify "File Version" matches expected values for DLLs' - - - powershell: | - # Check assembly versions. - # - # GOTCHA: This expects the Versions.props file having XML elements in a - # certain order. If the order changes, this check will fail! - # - # TODO: This also isn't checking the versions of the actual assemblies in - # the package, so it isn't terribly useful. - - [Xml] $versionprops = Get-Content -Path "tools/props/Versions.props" - $AssemblyFileVersion = $versionprops.Project.PropertyGroup[2].AssemblyFileVersion - $AssemblyVersion = $versionprops.Project.PropertyGroup[2].AssemblyVersion - - if($AssemblyFileVersion -eq $AssemblyVersion) - { - Write-Host AssemblyFileVersion: $AssemblyFileVersion should not be equal to: $AssemblyVersion - Exit -1 - } - displayName: 'Check "AssemblyFileVersion" is not same as "AssemblyVersion" in version.props' diff --git a/eng/pipelines/onebranch/scripts/compute-versions.ps1 b/eng/pipelines/onebranch/scripts/compute-versions.ps1 new file mode 100644 index 0000000000..4bce0b1495 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/compute-versions.ps1 @@ -0,0 +1,242 @@ +<# +.SYNOPSIS + Computes effective SqlClient and SqlServer package and file versions for OneBranch builds. + +.DESCRIPTION + Evaluates the canonical version properties through the GetVersionsSqlClient and + GetVersionsSqlServer targets in build.proj, selects the versions that the current pipeline run + should consume, and publishes those values as Azure DevOps job output variables. + + The SqlClient family always uses SqlClientNextVersion. Microsoft.SqlServer.Server uses + SqlServerNextVersion when BuildSqlServer is true and SqlServerPublishedVersion when it is false. + The published SqlServer version is needed when SqlServer is not built because downstream + SqlClient projects restore that existing package from NuGet. + + Package versions take a single shape, produced by Versions.props from the pipeline build number. + Preview versions carry the full build number after the prerelease suffix, such as + 1.2.3-preview1.26238.3. Stable versions are left exactly as declared in Versions.props, such as + 1.2.3, because released packages are not stamped with a build number. + + File versions are always four-part and always carry a build number in the fourth component, even + when the package version does not. Versions.props derives that component from the date segment of + BuildNumber, so a package version of 1.2.3-preview1.26238.3 has file version 1.2.3.26238, and a + stable package version of 1.2.3 still has file version 1.2.3.26238. That segment is date-coded, + so repeated runs on the same day share a file version even though their package versions differ. + + An unbuilt SqlServer package is never stamped with a build number because its effective version + must continue to identify the package that already exists on NuGet. + + The script emits these output variables for downstream stages: + - SqlClientPackageVersion + - SqlClientFileVersion + - SqlServerPackageVersion + - SqlServerFileVersion + - SqlClientApiScanVersion + - SqlServerApiScanVersion + +.PARAMETER ProjectPath + Absolute or relative path to the repository build.proj file. + +.PARAMETER BuildNumber + Pipeline build number in the form ., such as 26238.3. Versions.props appends it to + prerelease package versions and derives the file-version build number from its date segment. + +.PARAMETER BuildSqlServer + Whether this run builds Microsoft.SqlServer.Server. When false, the effective SqlServer package + version is its last published version and its file version is not consumed downstream. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing and specialized agent configurations. + +.EXAMPLE + ./compute-versions.ps1 ` + -ProjectPath ./build.proj ` + -BuildNumber 26238.3 ` + -BuildSqlServer $true + + Computes versions for a run that builds SqlServer. Prerelease package versions become + 7.1.0-preview3.26238.3 and file versions become 7.1.0.26238. + +.EXAMPLE + ./compute-versions.ps1 ` + -ProjectPath ./build.proj ` + -BuildNumber 26238.3 ` + -BuildSqlServer $false + + Stamps the SqlClient family versions while retaining SqlServerPublishedVersion for dependency + restore because SqlServer is not built in this run. + +.NOTES + File Name : compute-versions.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : compute-versions-stage.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Path to the repository build.proj file.")] + [ValidateScript({ Test-Path -LiteralPath $_ -PathType Leaf })] + [string]$ProjectPath, + + [Parameter(Mandatory = $true, HelpMessage = "Pipeline build number, such as 26238.3.")] + [ValidatePattern("^\d+\.\d+$")] + [string]$BuildNumber, + + [Parameter(Mandatory = $true, HelpMessage = "Whether Microsoft.SqlServer.Server is built in this run.")] + [bool]$BuildSqlServer, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +<# +.SYNOPSIS + Extracts the first value associated with a labeled GetVersions target output line. + +.PARAMETER Output + Output lines captured from dotnet build. + +.PARAMETER Label + Label prefix to find, such as PackageVersion or PublishedVersion. + +.OUTPUTS + The trimmed label value, or an empty string when the label is absent. +#> +function Get-LabeledValue { + param( + [string[]]$Output, + [string]$Label + ) + + $match = $Output | Select-String -Pattern "^\s*${Label}:\s*(.*?)\s*$" | Select-Object -First 1 + if ($null -eq $match) { + return "" + } + + return $match.Matches[0].Groups[1].Value.Trim() +} + +<# +.SYNOPSIS + Evaluates one canonical package family's versions through build.proj. + +.PARAMETER Label + GetVersions target suffix: SqlClient or SqlServer. + +.OUTPUTS + An object containing PackageVersion, FileVersion, and PublishedVersion. +#> +function Get-CanonicalVersions { + param( + [ValidateSet("SqlClient", "SqlServer")] + [string]$Label + ) + + $output = & $DotnetPath build $ProjectPath ` + -t:"GetVersions${Label}" -v:m -nologo -p:BuildNumber=$BuildNumber 2>&1 + if ($LASTEXITCODE -ne 0) { + throw ($output -join [Environment]::NewLine) + } + + $packageVersion = Get-LabeledValue -Output $output -Label "PackageVersion" + $fileVersion = Get-LabeledValue -Output $output -Label "FileVersion" + $publishedVersion = Get-LabeledValue -Output $output -Label "PublishedVersion" + if ([string]::IsNullOrWhiteSpace($packageVersion)) { + throw "Failed to extract PackageVersion for ${Label}.`n$($output -join [Environment]::NewLine)" + } + if ([string]::IsNullOrWhiteSpace($fileVersion)) { + throw "Failed to extract FileVersion for ${Label}.`n$($output -join [Environment]::NewLine)" + } + + [pscustomobject]@{ + PackageVersion = $packageVersion + FileVersion = $fileVersion + PublishedVersion = $publishedVersion + } +} + +<# +.SYNOPSIS + Extracts the major.minor components from a package version. + +.PARAMETER Version + Package version beginning with a numeric major.minor pair. + +.OUTPUTS + The major.minor version pair. +#> +function Get-MajorMinorVersion { + param( + [string]$Version + ) + + if ($Version -notmatch "^(\d+)\.(\d+)(?:\.|-|$)") { + throw "Unable to derive a major.minor version from package version '$Version'." + } + + return "$($Matches[1]).$($Matches[2])" +} + +<# +.SYNOPSIS + Emits an Azure DevOps job output variable for consumption by downstream stages. + +.PARAMETER Name + Output variable name. + +.PARAMETER Value + Output variable value. +#> +function Set-PipelineOutputVariable { + param( + [string]$Name, + [string]$Value + ) + + Write-Host "##vso[task.setvariable variable=${Name};isOutput=true]$Value" +} + +Write-Host "Extracting versions with build number $BuildNumber..." +$sqlClientVersions = Get-CanonicalVersions -Label "SqlClient" +$sqlServerVersions = Get-CanonicalVersions -Label "SqlServer" + +Write-Host " SqlClient: pkg=$($sqlClientVersions.PackageVersion) file=$($sqlClientVersions.FileVersion)" +Write-Host " SqlServer: pkg=$($sqlServerVersions.PackageVersion) file=$($sqlServerVersions.FileVersion) pub=$($sqlServerVersions.PublishedVersion)" + +$sqlClientPackageVersion = $sqlClientVersions.PackageVersion +$sqlClientFileVersion = $sqlClientVersions.FileVersion + +# An unbuilt SqlServer resolves to its published version, which no build job stamps, so it has no +# effective file version. +$sqlServerPackageVersion = if ($BuildSqlServer) { + $sqlServerVersions.PackageVersion +} else { + if ([string]::IsNullOrWhiteSpace($sqlServerVersions.PublishedVersion)) { + throw "GetVersionsSqlServer did not emit PublishedVersion for an unbuilt SqlServer dependency." + } + $sqlServerVersions.PublishedVersion +} +$sqlServerFileVersion = if ($BuildSqlServer) { $sqlServerVersions.FileVersion } else { "" } + +Write-Host "Effective versions:" +Write-Host " SqlClient (family): $sqlClientPackageVersion (file $sqlClientFileVersion)" +Write-Host " SqlServer: $sqlServerPackageVersion (file $sqlServerFileVersion)" + +$sqlClientApiScanVersion = Get-MajorMinorVersion -Version $sqlClientPackageVersion +$sqlServerApiScanVersion = Get-MajorMinorVersion -Version $sqlServerPackageVersion + +Write-Host "APIScan registration versions:" +Write-Host " SqlClient (family): $sqlClientApiScanVersion" +Write-Host " SqlServer: $sqlServerApiScanVersion" + +Set-PipelineOutputVariable -Name "SqlClientPackageVersion" -Value $sqlClientPackageVersion +Set-PipelineOutputVariable -Name "SqlClientFileVersion" -Value $sqlClientFileVersion +Set-PipelineOutputVariable -Name "SqlServerPackageVersion" -Value $sqlServerPackageVersion +Set-PipelineOutputVariable -Name "SqlServerFileVersion" -Value $sqlServerFileVersion +Set-PipelineOutputVariable -Name "SqlClientApiScanVersion" -Value $sqlClientApiScanVersion +Set-PipelineOutputVariable -Name "SqlServerApiScanVersion" -Value $sqlServerApiScanVersion diff --git a/eng/pipelines/onebranch/scripts/publish-symbols.ps1 b/eng/pipelines/onebranch/scripts/publish-symbols.ps1 new file mode 100644 index 0000000000..aeaa665a92 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/publish-symbols.ps1 @@ -0,0 +1,236 @@ +<# +.SYNOPSIS + Publishes symbols to the Microsoft symbol publishing service (SymWeb/MSDL). + +.DESCRIPTION + This script is Step 2 of the two-step symbols publishing process. It requests + the Symbols Publishing Pipeline to publish previously uploaded PDB files to + internal (SymWeb) and/or public (MSDL) Microsoft symbol servers. + + The two-step process: + Step 1 (PublishSymbols@2 task in publish-symbols-step.yml): + Uploads PDB files to the Azure DevOps symbol store under a unique + artifact name (SymbolsArtifactName). This stores the symbols but does + NOT make them available on SymWeb or MSDL. + + Step 2 (this script): + Calls the Symbols Publishing Pipeline REST API to request that the + uploaded symbols be published to the symbol servers. + + Step 2 depends on Step 1: the -ArtifactName parameter MUST match the + SymbolsArtifactName used by the PublishSymbols@2 upload task so that + both steps reference the same uploaded artifact. + + This script performs four sub-steps: + 1. Acquires a bearer token from Azure CLI for the symbol publishing service. + 2. Registers a unique request name with the publishing service. + 3. Submits the request to publish symbols to the specified servers. + 4. Queries the publishing status for confirmation. + + For more details on the Symbols Publishing Pipeline, see: + https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL + +.PARAMETER PublishServer + The hostname prefix of the symbol publishing service. This value is prepended to + '.trafficmanager.net' to construct the service base URL. + +.PARAMETER PublishTokenUri + The resource URI used to acquire a bearer token from Azure CLI + (via 'az account get-access-token --resource '). + +.PARAMETER PublishProjectName + The project name registered with the symbol publishing service (decided during onboarding). + +.PARAMETER ArtifactName + The name of the publishing request. This must match the SymbolsArtifactName used by + the PublishSymbols@2 upload task so that upload and publish reference the same artifact. + +.PARAMETER PublishToInternal + Whether to publish symbols to the internal symbol server. Defaults to $true. + +.PARAMETER PublishToPublic + Whether to publish symbols to the public symbol server. Defaults to $true. + +.EXAMPLE + .\publish-symbols.ps1 ` + -PublishServer "mysymbolserver" ` + -PublishTokenUri "https://login.microsoftonline.com/..." ` + -PublishProjectName "Microsoft.Data.SqlClient.SNI" ` + -ArtifactName "mds_symbols_MyProject_dotnet-sqlclient_main_7.0.0_abc123_1" + + Publishes symbols to both internal and public servers using the specified parameters. + +.EXAMPLE + .\publish-symbols.ps1 ` + -PublishServer "mysymbolserver" ` + -PublishTokenUri "https://login.microsoftonline.com/..." ` + -PublishProjectName "Microsoft.Data.SqlClient.SNI" ` + -ArtifactName "mds_symbols_MyProject_dotnet-sqlclient_main_7.0.0_abc123_2" ` + -PublishToPublic $false + + Publishes symbols to the internal server only (retry attempt 2). + +.NOTES + File Name : publish-symbols.ps1 + Requires : Azure CLI (az) must be installed and authenticated. + Called by : publish-symbols-step.yml (Azure Pipelines template) + + Publishing status codes returned by the service: + + PublishingStatus: + 0 - NotRequested: The request has not been requested to publish. + 1 - Submitted: The request is submitted to be published. + 2 - Processing: The request is still being processed. + 3 - Completed: Processing finished. Check PublishingResult for details. + + PublishingResult: + 0 - Pending: The request has not completed or has not been requested. + 1 - Succeeded: The request published successfully. + 2 - Failed: The request failed to publish. + 3 - Cancelled: The request was cancelled. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Hostname prefix of the symbol publishing service (prepended to .trafficmanager.net).")] + [ValidateNotNullOrEmpty()] + [string]$PublishServer, + + [Parameter(Mandatory = $true, HelpMessage = "Resource URI for acquiring a bearer token via Azure CLI.")] + [ValidateNotNullOrEmpty()] + [string]$PublishTokenUri, + + [Parameter(Mandatory = $true, HelpMessage = "Project name registered with the symbol publishing service.")] + [ValidateNotNullOrEmpty()] + [string]$PublishProjectName, + + [Parameter(Mandatory = $true, HelpMessage = "Artifact name for the publishing request (must match PublishSymbols@2 SymbolsArtifactName).")] + [ValidateNotNullOrEmpty()] + [string]$ArtifactName, + + [Parameter(Mandatory = $false, HelpMessage = "Publish symbols to the internal symbol server.")] + [bool]$PublishToInternal = $true, + + [Parameter(Mandatory = $false, HelpMessage = "Publish symbols to the public symbol server.")] + [bool]$PublishToPublic = $true +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# --- Log input parameters --- +Write-Host "=== Publish Symbols Parameters ===" +Write-Host "PublishServer: ${PublishServer}" +Write-Host "PublishTokenUri: ${PublishTokenUri}" +Write-Host "PublishProjectName: ${PublishProjectName}" +Write-Host "ArtifactName: ${ArtifactName}" +Write-Host "PublishToInternal: ${PublishToInternal}" +Write-Host "PublishToPublic: ${PublishToPublic}" +Write-Host "==================================" + +# --- Build request name and URLs --- +$requestName = ${ArtifactName} +$baseUrl = "https://${PublishServer}.trafficmanager.net/projects/${PublishProjectName}" +$registerUrl = "${baseUrl}/requests" +$requestUrl = "${baseUrl}/requests/${requestName}" + +Write-Host "=== Constructed URLs ===" +Write-Host "Request Name: ${requestName}" +Write-Host "Base URL: ${baseUrl}" +Write-Host "Register URL: ${registerUrl}" +Write-Host "Request URL: ${requestUrl}" +Write-Host "========================" + +# --- Step 1: Acquire token --- +Write-Host "> 1. Acquiring symbol publishing token..." +$symbolPublishingToken = az account get-access-token --resource ${PublishTokenUri} --query accessToken -o tsv +if ($LASTEXITCODE -ne 0) { + throw "Failed to acquire symbol publishing token via Azure CLI (exit code: ${LASTEXITCODE})." +} +if ($null -ne $symbolPublishingToken) { + $symbolPublishingToken = $symbolPublishingToken.Trim() +} +if ([string]::IsNullOrWhiteSpace($symbolPublishingToken)) { + throw "Failed to acquire symbol publishing token via Azure CLI: received an empty or whitespace-only access token." +} +Write-Host "> 1. Symbol publishing token acquired." + +$authHeaders = @{ Authorization = "Bearer ${symbolPublishingToken}" } + +# --- Step 2: Register request name --- +Write-Host "> 2. Registering request name..." +$requestNameRegistrationBody = @{ requestName = $requestName } | ConvertTo-Json -Compress +try { + Invoke-RestMethod -Method POST -Uri ${registerUrl} -Headers ${authHeaders} -ContentType "application/json" -Body ${requestNameRegistrationBody} +} catch { + throw "Failed to register request name. URI: ${registerUrl} | Body: ${requestNameRegistrationBody} | Error: $_" +} +Write-Host "> 2. Request name registered successfully." + +# --- Step 3: Publish symbols --- +Write-Host "> 3. Submitting request to publish symbols..." +$publishSymbolsBody = @{ + publishToInternalServer = $PublishToInternal + publishToPublicServer = $PublishToPublic +} | ConvertTo-Json -Compress +Write-Host "Publishing symbols request body: ${publishSymbolsBody}" +try { + Invoke-RestMethod -Method POST -Uri ${requestUrl} -Headers ${authHeaders} -ContentType "application/json" -Body ${publishSymbolsBody} +} catch { + throw "Failed to publish symbols. URI: ${requestUrl} | Body: ${publishSymbolsBody} | Error: $_" +} +Write-Host "> 3. Request to publish symbols submitted successfully." + +# --- Step 4: Check status --- +Write-Host "> 4. Checking the status of the request..." +try { + $status = Invoke-RestMethod -Method GET -Uri ${requestUrl} -Headers ${authHeaders} -ContentType "application/json" + $status +} catch { + throw "Failed to check request status. URI: ${requestUrl} | Error: $_" +} + +# Validate publishing results — fail the task when the service reports a terminal failure. +# PublishingResult: 0=Pending, 1=Succeeded, 2=Failed, 3=Cancelled +$resultLabels = @{ 0 = 'Pending'; 1 = 'Succeeded'; 2 = 'Failed'; 3 = 'Cancelled' } +$failures = @() + +if ($PublishToInternal) { + $internalResult = $status.publishToInternalServerResult + if ($null -ne $internalResult -and $internalResult -ge 2) { + $label = if ($resultLabels.ContainsKey([int]$internalResult)) { $resultLabels[[int]$internalResult] } else { "Unknown($internalResult)" } + $failures += "Internal server publishing result: ${label} (${internalResult})" + } +} + +if ($PublishToPublic) { + $publicResult = $status.publishToPublicServerResult + if ($null -ne $publicResult -and $publicResult -ge 2) { + $label = if ($resultLabels.ContainsKey([int]$publicResult)) { $resultLabels[[int]$publicResult] } else { "Unknown($publicResult)" } + $failures += "Public server publishing result: ${label} (${publicResult})" + } +} + +if ($failures.Count -gt 0) { + $failureMessage = $failures -join '; ' + throw "Symbol publishing reported a terminal failure. ${failureMessage}. URI: ${requestUrl}" +} + +Write-Host "> 4. Status check completed - no terminal failures detected." + +Write-Host "" +Write-Host "Use below tables to interpret the xxxServerStatus and xxxServerResult fields from the response." +Write-Host "" +Write-Host "PublishingStatus" +Write-Host "-----------------" +Write-Host "0 NotRequested - The request has not been requested to publish." +Write-Host "1 Submitted - The request is submitted to be published." +Write-Host "2 Processing - The request is still being processed." +Write-Host "3 Completed - Processing finished. Check PublishingResult for details." +Write-Host "" +Write-Host "PublishingResult" +Write-Host "-----------------" +Write-Host "0 Pending - The request has not completed or has not been requested." +Write-Host "1 Succeeded - The request published successfully." +Write-Host "2 Failed - The request failed to publish." +Write-Host "3 Cancelled - The request was cancelled." diff --git a/eng/pipelines/onebranch/scripts/tests/README.md b/eng/pipelines/onebranch/scripts/tests/README.md new file mode 100644 index 0000000000..593c7768d6 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/README.md @@ -0,0 +1,53 @@ +# OneBranch PowerShell Tests + +Pester tests for PowerShell scripts used by OneBranch pipeline steps. + +## Prerequisites + +- PowerShell 5.1+ or PowerShell 7+ +- [Pester v5](https://pester.dev/) (`Install-Module Pester -MinimumVersion 5.0 -Scope CurrentUser`) + +## Running the Tests + +From this directory: + +```powershell +Invoke-Pester ./publish-symbols.Tests.ps1 +``` + +Or from the repository root: + +```powershell +Invoke-Pester ./eng/pipelines/onebranch/scripts/tests/ +``` + +For detailed output: + +```powershell +Invoke-Pester ./publish-symbols.Tests.ps1 -Output Detailed +``` + +## Test Coverage + +| Area | What's tested | +| --------------------- | ---------------------------------------------------------------- | +| Version computation | Canonical output parsing, effective package selection, target version composition, and failures | +| Localization validation | Missing, obsolete, or empty strings, English-value matches, and culture-specific allowlisting | +| Parameter validation | Empty strings rejected for all mandatory parameters | +| URL construction | Base URL, register URL, request URL built from parameters | +| Request bodies | Registration body, default publish flags, flag overrides | +| Error handling | Token failure, registration failure, publish failure, status failure — all verify expanded URI in error message | +| Status validation | Detects Failed/Cancelled results, respects PublishToInternal/PublishToPublic flags, passes on Succeeded/Pending | +| Package validation | Wildcard vs per-id version expectations, SqlServer omitted when unbuilt, gate tokens, report written before gating, exit-code handling | +| Package signatures | Every package and symbol package verified, all failures reported before throwing | +| Assembly signatures | Package expansion, native binaries under `runtimes/` included, stale expansions replaced, all unsigned assemblies reported | + +## Notes + +- All external calls (`az`, `Invoke-RestMethod`) are mocked — no network access or Azure credentials are required. +- Script-level version tests mock `dotnet`; package-composition tests invoke the real MSBuild + `GetVersionsSqlClient` and `GetVersionsSqlServer` targets. +- `Get-AuthenticodeSignature` is Windows-only, so the assembly-signature tests declare a stub when + it is absent. Only the signature lookup is substituted; package expansion and reporting run for + real against packages built in the test's temporary directory. +- Tests validate scripts in the parent directory relative to this directory. diff --git a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 new file mode 100644 index 0000000000..7a9e9779cb --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 @@ -0,0 +1,401 @@ +<# +.SYNOPSIS + Pester tests for compute-versions.ps1. +#> + +BeforeAll { + $script:repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..' '..') + $scriptPath = Join-Path $PSScriptRoot '..' 'compute-versions.ps1' + $buildProjectPath = Resolve-Path (Join-Path $script:repoRoot 'build.proj') + $projectPath = Join-Path $TestDrive 'build.proj' + Set-Content -LiteralPath $projectPath -Value '' + + # An arbitrary but well-formed pipeline build number, used only by the tests that exercise the + # path which consumes one. The script never reads the ambient build number; the pipeline passes + # $(Build.BuildNumber) as a parameter, so any well-formed value works here and a fixed one keeps + # the expected output deterministic. Assertions derive from these rather than repeating literals + # so the relationship between the two is visible. + $script:testBuildNumber = '26238.3' + $script:testBuildNumberPattern = [regex]::Escape($script:testBuildNumber) + $script:testFileVersionBuildNumber = $script:testBuildNumber.Split('.')[0] + + function Invoke-ComputeVersions { + param( + [string]$BuildNumber = $script:testBuildNumber, + [bool]$BuildSqlServer = $true + ) + + & $scriptPath ` + -ProjectPath $projectPath ` + -BuildNumber $BuildNumber ` + -BuildSqlServer $BuildSqlServer *>&1 | Out-String + } + + # Alternates between the SqlClient and SqlServer GetVersions targets, which the script always + # invokes in that order. The versions returned here are already stamped, because Versions.props + # applies the build number before the script ever sees them. + function Set-DotnetMock { + param( + [string]$SqlClientPackageVersion = "7.1.0-preview3.$script:testBuildNumber", + [string]$SqlServerPackageVersion = "1.1.0-preview1.$script:testBuildNumber" + ) + + $global:computeVersionsDotnetCallCount = 0 + Mock -CommandName 'dotnet' -MockWith { + $global:LASTEXITCODE = 0 + $global:computeVersionsDotnetCallCount++ + if ($global:computeVersionsDotnetCallCount % 2 -eq 1) { + return @( + " PackageVersion: $SqlClientPackageVersion" + ' FileVersion: 7.1.0.26238' + ' PublishedVersion: 7.0.0' + ) + } + + return @( + " PackageVersion: $SqlServerPackageVersion" + ' FileVersion: 1.1.0.26238' + ' PublishedVersion: 1.0.0' + ) + }.GetNewClosure() + } + + function Set-SuccessfulDotnetMock { + Set-DotnetMock + } + + function Invoke-VersionTarget { + param( + [Parameter(Mandatory)] + [string]$Target, + + [Parameter(Mandatory)] + [string]$NextVersionProperty, + + [Parameter(Mandatory)] + [string]$BaseVersion, + + [string]$BuildSuffix + ) + + $arguments = @( + 'build' + $buildProjectPath + "-t:$Target" + '-v:m' + '-nologo' + "-p:BuildNumber=$script:testBuildNumber" + "-p:$NextVersionProperty=$BaseVersion" + ) + if ($BuildSuffix) { + $arguments += "-p:BuildSuffix=$BuildSuffix" + } + + $output = & dotnet @arguments 2>&1 | Out-String + if ($LASTEXITCODE -ne 0) { + throw "$Target failed with exit code ${LASTEXITCODE}:`n$output" + } + + $output + } + + # Drives PrepareForBuild rather than the validation target directly, because the hook point is + # itself the thing under test: a check wired after the compile would pass a direct invocation. + # An explicit target framework is required, as PrepareForBuild is not valid on the outer + # cross-targeting build. + function Invoke-VersionValidation { + param( + [Parameter(Mandatory)] + [string]$ProjectPath, + + [Parameter(Mandatory)] + [string]$TargetFramework, + + [string[]]$Properties = @() + ) + + $arguments = @( + 'build' + $ProjectPath + '-f' + $TargetFramework + '-t:PrepareForBuild' + '-v:m' + '-nologo' + ) + $Properties + + $output = & dotnet @arguments 2>&1 | Out-String + [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } + } + + function Invoke-BuildProjTarget { + param( + [Parameter(Mandatory)] + [string]$Target, + + [string[]]$Properties = @() + ) + + $arguments = @( + 'build' + $buildProjectPath + "-t:$Target" + '-v:m' + '-nologo' + ) + $Properties + + $output = & dotnet @arguments 2>&1 | Out-String + [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } + } +} + +AfterAll { + Remove-Variable -Name 'computeVersionsDotnetCallCount' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'compute-versions.ps1 Effective Versions' { + BeforeEach { + Set-SuccessfulDotnetMock + } + + It 'forwards the stamped prerelease versions from Versions.props' { + $output = Invoke-ComputeVersions + + $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" + $output | Should -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0-preview1\.$script:testBuildNumberPattern" + $output | Should -Match 'SqlClientApiScanVersion;isOutput=true]7\.1' + $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.1' + $output | Should -Match 'APIScan registration versions:\s+SqlClient \(family\): 7\.1\s+SqlServer:\s+1\.1' + $output | Should -Match 'SqlClientFileVersion;isOutput=true]7\.1\.0\.26238' + $output | Should -Match 'SqlServerFileVersion;isOutput=true]1\.1\.0\.26238' + } + + It 'retains the published SqlServer package when SqlServer is not built' { + $output = Invoke-ComputeVersions -BuildSqlServer $false + + $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" + $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0' + $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.0' + $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.0\.0[\.-]$script:testFileVersionBuildNumber" + + # An unbuilt SqlServer is never stamped, so it has no effective file version. + $output | Should -Match 'SqlServerFileVersion;isOutput=true](\r?\n|$)' + } + + It 'forwards unstamped non-preview package versions' { + Set-DotnetMock -SqlClientPackageVersion '7.1.0' -SqlServerPackageVersion '1.1.0' + + $output = Invoke-ComputeVersions + + $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0(\r?\n|$)' + $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0(\r?\n|$)' + $output | Should -Not -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0[\.-]$script:testFileVersionBuildNumber" + $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0[\.-]$script:testFileVersionBuildNumber" + + # The file version is still stamped so every build produces a date-encoded file version even + # for non-preview releases. + $output | Should -Match 'SqlClientFileVersion;isOutput=true]7\.1\.0\.26238' + } +} + +Describe 'GetVersions target package composition' { + It ' composes package and file versions' -ForEach @( + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0'; BuildSuffix = ''; ExpectedPackageVersion = '7.1.0' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a stable base without a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0'; BuildSuffix = 'ci'; ExpectedPackageVersion = '7.1.0-ci.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a stable base with a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0-preview3'; BuildSuffix = ''; ExpectedPackageVersion = '7.1.0-preview3.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a prerelease base without a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0-preview3'; BuildSuffix = 'ci'; ExpectedPackageVersion = '7.1.0-preview3-ci.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a prerelease base with a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0'; BuildSuffix = ''; ExpectedPackageVersion = '1.1.0' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a stable base without a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0'; BuildSuffix = 'ci'; ExpectedPackageVersion = '1.1.0-ci.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a stable base with a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0-preview1'; BuildSuffix = ''; ExpectedPackageVersion = '1.1.0-preview1.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a prerelease base without a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0-preview1'; BuildSuffix = 'ci'; ExpectedPackageVersion = '1.1.0-preview1-ci.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a prerelease base with a suffix' + } + ) { + $output = Invoke-VersionTarget ` + -Target $Target ` + -NextVersionProperty $NextVersionProperty ` + -BaseVersion $BaseVersion ` + -BuildSuffix $BuildSuffix + + $output | Should -Match "PackageVersion:\s+$([regex]::Escape($ExpectedPackageVersion))(\r?\n|$)" + $output | Should -Match "FileVersion:\s+$([regex]::Escape($ExpectedFileVersion))(\r?\n|$)" + } +} + +Describe 'File version component validation' { + It 'rejects a four-part ' -ForEach @( + @{ + Product = 'SqlClient'; Property = 'SqlClientPackageVersion' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + Properties = @('-p:SqlClientPackageVersion=7.1.0.123') + ExpectedFileVersion = '7.1.0.123.0' + } + @{ + Product = 'SqlClient'; Property = 'SqlClientNextVersion' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + Properties = @('-p:SqlClientNextVersion=7.1.0.123', '-p:BuildNumber=1234') + ExpectedFileVersion = '7.1.0.123.1234' + } + @{ + Product = 'SqlServer'; Property = 'SqlServerPackageVersion' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + Properties = @('-p:SqlServerPackageVersion=1.1.0.123') + ExpectedFileVersion = '1.1.0.123.0' + } + @{ + Product = 'SqlServer'; Property = 'SqlServerNextVersion' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + Properties = @('-p:SqlServerNextVersion=1.1.0.123', '-p:BuildNumber=1234') + ExpectedFileVersion = '1.1.0.123.1234' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties $Properties + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '$ExpectedFileVersion' is not a four-part numeric version")) + } + + It 'rejects an externally supplied file version for ' -ForEach @( + @{ + Product = 'SqlClient'; Description = 'short' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + FileVersion = '1.2' + } + @{ + Product = 'SqlClient'; Description = 'non-numeric' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + FileVersion = 'abc' + } + @{ + Product = 'SqlServer'; Description = 'short' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + FileVersion = '1.2' + } + @{ + Product = 'SqlServer'; Description = 'non-numeric' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + FileVersion = 'abc' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties @("-p:${Product}FileVersion=$FileVersion") + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '$FileVersion' is not a four-part numeric version")) + } + + It 'accepts the declared version' -ForEach @( + @{ + Product = 'SqlClient' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + } + @{ + Product = 'SqlServer' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties @("-p:BuildNumber=$script:testBuildNumber") + + $result.ExitCode | Should -Be 0 + } +} + +Describe 'build.proj file version wrappers' { + # A malformed value is used so the leaf project reports it by name, which proves the wrapper + # forwarded it verbatim without paying for a full compile. + It 'forwards through ' -ForEach @( + @{ Product = 'SqlClient'; Wrapper = 'FileVersionSqlClient'; Target = 'BuildLogging' } + @{ Product = 'SqlServer'; Wrapper = 'FileVersionSqlServer'; Target = 'BuildSqlServer' } + ) { + $result = Invoke-BuildProjTarget -Target $Target -Properties @("-p:$Wrapper=1.2") + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '1.2' is not a four-part numeric version")) + } +} + +Describe 'compute-versions.ps1 Error Handling' { + It 'rejects a malformed build number' { + { Invoke-ComputeVersions -BuildNumber 'not-a-build-number' } | Should -Throw + } + + It 'requires a build number' { + # Bound as empty rather than omitted; omitting a mandatory parameter prompts interactively. + { Invoke-ComputeVersions -BuildNumber '' } | Should -Throw + } + + It 'throws when a GetVersions target fails' { + Mock -CommandName 'dotnet' -MockWith { + $global:LASTEXITCODE = 1 + return 'simulated target failure' + } + + { Invoke-ComputeVersions } | Should -Throw '*simulated target failure*' + } + + It 'throws when required version labels are absent' { + Mock -CommandName 'dotnet' -MockWith { + $global:LASTEXITCODE = 0 + return 'Build succeeded without version labels' + } + + { Invoke-ComputeVersions } | Should -Throw '*Failed to extract PackageVersion*' + } + + It 'throws when a FileVersion label is absent' { + Mock -CommandName 'dotnet' -MockWith { + $global:LASTEXITCODE = 0 + return 'PackageVersion: 7.1.0-preview3.26238.3' + } + + { Invoke-ComputeVersions } | Should -Throw '*Failed to extract FileVersion*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/publish-symbols.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/publish-symbols.Tests.ps1 new file mode 100644 index 0000000000..4d16d54e96 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/publish-symbols.Tests.ps1 @@ -0,0 +1,356 @@ +<# +.SYNOPSIS + Pester tests for publish-symbols.ps1 +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'publish-symbols.ps1' +} + +AfterAll { + # Clean up global variables used by mocks + Remove-Variable -Name 'restCalls' -Scope Global -ErrorAction SilentlyContinue + Remove-Variable -Name 'mockCallCount' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'publish-symbols.ps1 Parameter Validation' { + + It 'Should reject an empty PublishServer' { + { & $scriptPath -PublishServer '' -PublishTokenUri 'https://token' -PublishProjectName 'proj' -ArtifactName 'art' } | + Should -Throw + } + + It 'Should reject an empty PublishTokenUri' { + { & $scriptPath -PublishServer 'server' -PublishTokenUri '' -PublishProjectName 'proj' -ArtifactName 'art' } | + Should -Throw + } + + It 'Should reject an empty PublishProjectName' { + { & $scriptPath -PublishServer 'server' -PublishTokenUri 'https://token' -PublishProjectName '' -ArtifactName 'art' } | + Should -Throw + } + + It 'Should reject an empty ArtifactName' { + { & $scriptPath -PublishServer 'server' -PublishTokenUri 'https://token' -PublishProjectName 'proj' -ArtifactName '' } | + Should -Throw + } +} + +Describe 'publish-symbols.ps1 URL Construction' { + + BeforeAll { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token-12345' } + + $global:restCalls = @() + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:restCalls += @{ + Method = $Method + Uri = $Uri + Body = $Body + } + return @{ publishToInternalServerStatus = 0; publishToPublicServerStatus = 0; publishToInternalServerResult = 0; publishToPublicServerResult = 0 } + } + } + + BeforeEach { + $global:restCalls = @() + } + + It 'Should construct the correct base URL from PublishServer and PublishProjectName' { + & $scriptPath ` + -PublishServer 'myserver' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'My.Project' ` + -ArtifactName 'test_artifact' + + $global:restCalls.Count | Should -Be 3 + + # Registration call + $global:restCalls[0].Uri | Should -Be 'https://myserver.trafficmanager.net/projects/My.Project/requests' + $global:restCalls[0].Method | Should -Be 'POST' + + # Publish call + $global:restCalls[1].Uri | Should -Be 'https://myserver.trafficmanager.net/projects/My.Project/requests/test_artifact' + $global:restCalls[1].Method | Should -Be 'POST' + + # Status call + $global:restCalls[2].Uri | Should -Be 'https://myserver.trafficmanager.net/projects/My.Project/requests/test_artifact' + $global:restCalls[2].Method | Should -Be 'GET' + } + + It 'Should use ArtifactName directly as the request name' { + & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'myartifact_3' + + $global:restCalls[1].Uri | Should -BeLike '*myartifact_3' + $global:restCalls[2].Uri | Should -BeLike '*myartifact_3' + } +} + +Describe 'publish-symbols.ps1 Request Bodies' { + + BeforeAll { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token-12345' } + + $global:restCalls = @() + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:restCalls += @{ + Method = $Method + Uri = $Uri + Body = $Body + } + return @{ publishToInternalServerStatus = 0; publishToPublicServerStatus = 0; publishToInternalServerResult = 0; publishToPublicServerResult = 0 } + } + } + + BeforeEach { + $global:restCalls = @() + } + + It 'Should send the correct request name in the registration body' { + & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'my_artifact_1' + + $body = $global:restCalls[0].Body | ConvertFrom-Json + $body.requestName | Should -Be 'my_artifact_1' + } + + It 'Should default to publishing to both internal and public servers' { + & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' + + $body = $global:restCalls[1].Body | ConvertFrom-Json + $body.publishToInternalServer | Should -Be $true + $body.publishToPublicServer | Should -Be $true + } + + It 'Should respect PublishToInternal=false' { + & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' ` + -PublishToInternal $false + + $body = $global:restCalls[1].Body | ConvertFrom-Json + $body.publishToInternalServer | Should -Be $false + $body.publishToPublicServer | Should -Be $true + } + + It 'Should respect PublishToPublic=false' { + & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' ` + -PublishToPublic $false + + $body = $global:restCalls[1].Body | ConvertFrom-Json + $body.publishToInternalServer | Should -Be $true + $body.publishToPublicServer | Should -Be $false + } +} + +Describe 'publish-symbols.ps1 Error Handling' { + + It 'Should throw when token acquisition fails' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 1; return '' } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*token*' + } + + It 'Should throw when token is empty' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return ' ' } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*empty*' + } + + It 'Should throw with URI details when registration fails' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + Mock -CommandName 'Invoke-RestMethod' -MockWith { throw "Connection refused" } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*Failed to register*' + } + + It 'Should throw with URI details when publish fails' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -eq 1) { return @{} } + throw "Service unavailable" + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*Failed to publish*' + } + + It 'Should throw with URI details when status check fails' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + throw "Timeout" + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*Failed to check*' + } +} + +Describe 'publish-symbols.ps1 Status Failure Detection' { + + It 'Should throw when internal server result is Failed (2)' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 2; publishToPublicServerResult = 0 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*terminal failure*Internal server*Failed*' + } + + It 'Should throw when public server result is Cancelled (3)' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 1; publishToPublicServerResult = 3 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*terminal failure*Public server*Cancelled*' + } + + It 'Should throw when both servers report failure' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 2; publishToPublicServerResult = 3 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Throw '*terminal failure*Internal server*Public server*' + } + + It 'Should not throw when both servers report Succeeded (1)' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 1; publishToPublicServerResult = 1 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Not -Throw + } + + It 'Should not throw when results are Pending (0)' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 0; publishToPublicServerResult = 0 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' } | + Should -Not -Throw + } + + It 'Should not check internal result when PublishToInternal is false' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 2; publishToPublicServerResult = 1 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' ` + -PublishToInternal $false } | + Should -Not -Throw + } + + It 'Should not check public result when PublishToPublic is false' { + Mock -CommandName 'az' -MockWith { $global:LASTEXITCODE = 0; return 'fake-token' } + $global:mockCallCount = 0 + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:mockCallCount++ + if ($global:mockCallCount -le 2) { return @{} } + return @{ publishToInternalServerResult = 1; publishToPublicServerResult = 2 } + } + + { & $scriptPath ` + -PublishServer 'srv' ` + -PublishTokenUri 'https://token-uri' ` + -PublishProjectName 'proj' ` + -ArtifactName 'art' ` + -PublishToPublic $false } | + Should -Not -Throw + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 new file mode 100644 index 0000000000..5f27b883d3 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 @@ -0,0 +1,190 @@ +<# +.SYNOPSIS + Pester tests for validate-localization.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'validate-localization.ps1' + + function Set-ResourceFile { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][hashtable]$Strings + ) + + $document = [System.Xml.XmlDocument]::new() + $root = $document.CreateElement('root') + $null = $document.AppendChild($root) + foreach ($entry in $Strings.GetEnumerator()) { + $data = $document.CreateElement('data') + $data.SetAttribute('name', $entry.Key) + + $value = $document.CreateElement('value') + $value.InnerText = $entry.Value + $null = $data.AppendChild($value) + $null = $root.AppendChild($data) + } + + $document.Save($Path) + } + + function New-ResourcesDirectory { + $path = Join-Path $TestDrive ([guid]::NewGuid().ToString('n')) + New-Item -ItemType Directory -Path $path | Out-Null + return $path + } +} + +Describe 'validate-localization.ps1' { + It 'accepts complete localized files with translated values' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour'; Farewell = 'Au revoir' } + + { & $scriptPath -ResourcesDirectory $resources } | Should -Not -Throw + } + + It 'fails when a localized file is missing an English key' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hallo' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when a localized value matches a non-empty English value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.ja.resx') @{ Greeting = 'Hello'; Unused = '' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when no localized resource files exist' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*No localized Strings.*.resx files were found*' + } + + It 'fails when a non-empty English string has an empty localized value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.es.resx') @{ Greeting = ' ' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'accepts empty localized values when the English value is also empty' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.ko.resx') @{ Unused = '' } + + { & $scriptPath -ResourcesDirectory $resources } | Should -Not -Throw + } + + It 'fails when a resource data element has no value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + [xml]$localized = Get-Content -LiteralPath (Join-Path $resources 'Strings.fr.resx') + $valueNode = $localized.SelectSingleNode('/root/data/value') + $null = $valueNode.ParentNode.RemoveChild($valueNode) + $localized.Save((Join-Path $resources 'Strings.fr.resx')) + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*contains a element without a name or value*' + } + + It 'accepts an approved English-value match from the allowlist' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Not -Throw + } + + It 'does not allowlist a missing localized key' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Farewell') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects allowlist for unknown resource keys' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unknown') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'scopes approved English-value matches to one localized file' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + @{ AllowedEnglishValueMatches = @{ 'Strings.de.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when a localized file contains a key absent from Strings.resx' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour'; Obsolete = 'Ancien' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects an allowlist entry after the localized value is translated' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects allowlist entries for empty English values' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Unused = '' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unused') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*does not have a non-empty English value*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 new file mode 100644 index 0000000000..626dd2dca9 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 @@ -0,0 +1,188 @@ +<# +.SYNOPSIS + Pester tests for validate-packages.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'validate-packages.ps1' + + # Stands in for the built PackageValidator.dll; the script only checks that it exists. + $script:validatorPath = Join-Path $TestDrive 'PackageValidator.dll' + Set-Content -LiteralPath $script:validatorPath -Value 'stub' + + $script:packagesPath = Join-Path $TestDrive 'packages' + New-Item -ItemType Directory -Force -Path $script:packagesPath | Out-Null + Set-Content -LiteralPath (Join-Path $script:packagesPath 'Microsoft.Data.SqlClient.7.1.0.nupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'Microsoft.SqlServer.Server.1.1.0.nupkg') -Value 'stub' + + $script:reportPath = Join-Path $TestDrive 'out' 'report.json' + + function Invoke-ValidatePackages { + param( + [string]$PackagesPath = $script:packagesPath, + [string]$ValidatorPath = $script:validatorPath, + [string]$SqlClientPackageVersion = '7.1.0-preview3.26238.3', + [string]$SqlClientFileVersion = '7.1.0.26238', + [string]$SqlServerPackageVersion = '', + [string]$SqlServerFileVersion = '', + [string[]]$FailOn = @('error') + ) + + & $scriptPath ` + -ValidatorPath $ValidatorPath ` + -PackagesPath $PackagesPath ` + -ReportPath $script:reportPath ` + -SqlClientPackageVersion $SqlClientPackageVersion ` + -SqlClientFileVersion $SqlClientFileVersion ` + -SqlServerPackageVersion $SqlServerPackageVersion ` + -SqlServerFileVersion $SqlServerFileVersion ` + -FailOn $FailOn ` + -DotnetPath 'dotnet' *>&1 | Out-String + } + + # Captures the arguments of each invocation so tests can assert on what the validator was + # asked to do, and controls the exit code of each run. + function Set-DotnetMock { + param( + [int]$GateExitCode = 0, + [int]$ReportExitCode = 0 + ) + + $global:validatePackagesInvocations = @() + Mock -CommandName 'dotnet' -MockWith { + $global:validatePackagesInvocations += , @($args) + # The first run carries --json and never gates; the second applies the gate. + if ($args -contains '--json') { + $global:LASTEXITCODE = $ReportExitCode + return '{ "packages": [], "summary": {} }' + } + + $global:LASTEXITCODE = $GateExitCode + return 'validator output' + }.GetNewClosure() + } +} + +AfterAll { + Remove-Variable -Name 'validatePackagesInvocations' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'validate-packages.ps1 Expectations' { + BeforeEach { + Set-DotnetMock + } + + It 'applies the SqlClient family versions as wildcard expectations' { + Invoke-ValidatePackages | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $gateArgs | Should -Contain '*=7.1.0-preview3.26238.3' + $gateArgs | Should -Contain '*=7.1.0.26238' + } + + It 'omits SqlServer expectations when its versions are not supplied' { + Invoke-ValidatePackages | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + ($gateArgs -join ' ') | Should -Not -Match 'Microsoft\.SqlServer\.Server=' + } + + It 'adds SqlServer expectations as a per-id override when supplied' { + Invoke-ValidatePackages -SqlServerPackageVersion '1.1.0-preview1.26238.3' -SqlServerFileVersion '1.1.0.26238' | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $gateArgs | Should -Contain 'Microsoft.SqlServer.Server=1.1.0-preview1.26238.3' + $gateArgs | Should -Contain 'Microsoft.SqlServer.Server=1.1.0.26238' + } + + It 'passes each gate token as its own --fail-on argument' { + Invoke-ValidatePackages -FailOn @('error', 'missing-symbols', 'package-unsigned') | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $joined = $gateArgs -join ' ' + $joined | Should -Match '--fail-on error' + $joined | Should -Match '--fail-on missing-symbols' + $joined | Should -Match '--fail-on package-unsigned' + } + + It 'splits a single comma-separated gate token, as an Azure Pipelines argument line supplies it' { + Invoke-ValidatePackages -FailOn 'error, missing-symbols' | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $joined = $gateArgs -join ' ' + $joined | Should -Match '--fail-on error' + $joined | Should -Match '--fail-on missing-symbols' + $joined | Should -Not -Match 'error,' + } + + It 'writes the JSON report before applying the gate' { + Invoke-ValidatePackages | Out-Null + + # The reporting run must come first so the report survives a failing gate. + $firstInvocation = $global:validatePackagesInvocations | Select-Object -First 1 + $firstInvocation | Should -Contain '--json' + Test-Path -LiteralPath $script:reportPath | Should -BeTrue + } + + It 'does not gate the reporting run' { + Invoke-ValidatePackages -FailOn @('error') | Out-Null + + $reportArgs = $global:validatePackagesInvocations | Where-Object { $_ -contains '--json' } | Select-Object -First 1 + ($reportArgs -join ' ') | Should -Not -Match '--fail-on' + } +} + +Describe 'validate-packages.ps1 Exit Codes' { + It 'succeeds when the validator reports no gating findings' { + Set-DotnetMock -GateExitCode 0 + + $output = Invoke-ValidatePackages + $output | Should -Match 'Package validation passed' + } + + It 'fails when a gate is tripped' { + Set-DotnetMock -GateExitCode 2 + + { Invoke-ValidatePackages -FailOn @('error', 'missing-symbols') } | + Should -Throw '*matched the gate (error, missing-symbols)*' + } + + It 'reports an unexpected validator failure distinctly from a tripped gate' { + Set-DotnetMock -GateExitCode 1 + + { Invoke-ValidatePackages } | Should -Throw '*exited unexpectedly with code 1*' + } + + It 'fails the reporting run before gating so the real cause is not obscured' { + Set-DotnetMock -ReportExitCode 1 + + { Invoke-ValidatePackages } | Should -Throw '*failed while writing the JSON report (exit code 1)*' + + # The gating run must not have been reached. + $global:validatePackagesInvocations.Count | Should -Be 1 + } +} + +Describe 'validate-packages.ps1 Error Handling' { + BeforeEach { + Set-DotnetMock + } + + It 'throws when the validator is missing' { + { Invoke-ValidatePackages -ValidatorPath (Join-Path $TestDrive 'absent.dll') } | + Should -Throw '*PackageValidator was not found*' + } + + It 'throws when no packages are found' { + $empty = Join-Path $TestDrive 'empty' + New-Item -ItemType Directory -Force -Path $empty | Out-Null + + { Invoke-ValidatePackages -PackagesPath $empty } | Should -Throw '*No .nupkg files were found*' + } + + It 'rejects a half-supplied SqlServer expectation' { + # Supplying only one would assert a package version without its file version. + { Invoke-ValidatePackages -SqlServerPackageVersion '1.1.0' } | + Should -Throw '*must be supplied together*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 new file mode 100644 index 0000000000..4d4946a5fb --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 @@ -0,0 +1,149 @@ +<# +.SYNOPSIS + Pester tests for verify-assembly-signatures.ps1. + +.NOTES + Get-AuthenticodeSignature is a Windows-only cmdlet, so a stub is declared when it is absent. + This lets the tests run on any platform while still exercising the script's real expansion and + reporting logic; only the signature lookup itself is substituted. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'verify-assembly-signatures.ps1' + + if (-not (Get-Command 'Get-AuthenticodeSignature' -ErrorAction SilentlyContinue)) { + function Get-AuthenticodeSignature { + param([Parameter(Mandatory = $true)][string[]]$FilePath) + throw 'stub must be mocked' + } + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + # Builds a real .nupkg so the script's expansion path is genuinely exercised. + function New-TestPackage { + param( + [string]$Name, + [string[]]$AssemblyPaths = @('lib/net8.0/Test.dll'), + [string[]]$OtherPaths = @() + ) + + $staging = Join-Path $TestDrive "staging-$Name" + if (Test-Path -LiteralPath $staging) { Remove-Item -LiteralPath $staging -Recurse -Force } + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + foreach ($relative in ($AssemblyPaths + $OtherPaths)) { + $full = Join-Path $staging $relative + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $full) | Out-Null + Set-Content -LiteralPath $full -Value 'stub' + } + + $packagePath = Join-Path $script:packagesPath "$Name.nupkg" + if (Test-Path -LiteralPath $packagePath) { Remove-Item -LiteralPath $packagePath -Force } + [System.IO.Compression.ZipFile]::CreateFromDirectory($staging, $packagePath) + return $packagePath + } + + function Invoke-VerifyAssemblySignatures { + param( + [string]$PackagesPath = $script:packagesPath, + [string]$ExtractPath = $script:extractPath + ) + + & $scriptPath -PackagesPath $PackagesPath -ExtractPath $ExtractPath *>&1 | Out-String + } +} + +AfterAll { + Remove-Variable -Name 'verifyAssemblySeen' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'verify-assembly-signatures.ps1' { + BeforeEach { + $script:packagesPath = Join-Path $TestDrive 'packages' + $script:extractPath = Join-Path $TestDrive 'extract' + foreach ($path in @($script:packagesPath, $script:extractPath)) { + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force } + New-Item -ItemType Directory -Force -Path $path | Out-Null + } + } + + It 'expands packages and verifies every assembly they contain' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') | Out-Null + New-TestPackage -Name 'PackageTwo' -AssemblyPaths @('lib/net8.0/Two.dll', 'runtimes/win-x64/native/sni.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 3 assemblies are Authenticode signed' + } + + It 'finds native binaries under runtimes, not just managed assemblies' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('runtimes/win-arm64/native/sni.dll') | Out-Null + $global:verifyAssemblySeen = @() + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $global:verifyAssemblySeen = $FilePath + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + Invoke-VerifyAssemblySignatures | Out-Null + ($global:verifyAssemblySeen -join ';') | Should -Match 'sni\.dll' + } + + It 'ignores non-assembly content' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') -OtherPaths @('README.md', 'lib/net8.0/One.xml') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 1 assemblies are Authenticode signed' + } + + It 'fails when an assembly is not validly signed' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Two.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $index = 0 + $FilePath | ForEach-Object { + $status = if ($index -eq 0) { 'Valid' } else { 'NotSigned' } + $index++ + [pscustomobject]@{ Path = $_; Status = $status } + } + } + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*failed for 1 of 2 assemblies*' + } + + It 'reports every unsigned assembly rather than stopping at the first' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Two.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'NotSigned' } } + } + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*failed for 2 of 2 assemblies*' + } + + It 'replaces a previous expansion so stale content cannot be verified' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Stale.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + Invoke-VerifyAssemblySignatures | Out-Null + + # Repack the same package id with fewer assemblies; the stale one must not linger. + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') | Out-Null + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 1 assemblies are Authenticode signed' + } + + It 'throws when no packages are found' { + { Invoke-VerifyAssemblySignatures } | Should -Throw '*No .nupkg files were found*' + } + + It 'throws when packages contain no assemblies' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @() -OtherPaths @('README.md') | Out-Null + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*No assemblies were found*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 new file mode 100644 index 0000000000..cadc1831a4 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 @@ -0,0 +1,87 @@ +<# +.SYNOPSIS + Pester tests for verify-package-signatures.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'verify-package-signatures.ps1' + + $script:packagesPath = Join-Path $TestDrive 'packages' + New-Item -ItemType Directory -Force -Path (Join-Path $script:packagesPath 'SqlClient') | Out-Null + New-Item -ItemType Directory -Force -Path (Join-Path $script:packagesPath 'SqlServer') | Out-Null + + # Symbol packages are signed too, so both extensions must be picked up, and the nested layout + # mirrors how each artifact is downloaded into its own subdirectory. + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlClient' 'Microsoft.Data.SqlClient.7.1.0.nupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlClient' 'Microsoft.Data.SqlClient.7.1.0.snupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlServer' 'Microsoft.SqlServer.Server.1.1.0.nupkg') -Value 'stub' + + function Invoke-VerifyPackageSignatures { + param([string]$PackagesPath = $script:packagesPath) + + & $scriptPath -PackagesPath $PackagesPath -DotnetPath 'dotnet' *>&1 | Out-String + } + + # Fails verification only for packages whose name matches, so tests can make a subset unsigned. + function Set-DotnetMock { + param([string]$FailPattern = '') + + $global:verifyPackageInvocations = @() + Mock -CommandName 'dotnet' -MockWith { + $global:verifyPackageInvocations += , @($args) + $target = $args[-1] + if ($FailPattern -and $target -match $FailPattern) { + $global:LASTEXITCODE = 1 + return "unsigned" + } + + $global:LASTEXITCODE = 0 + return "verified" + }.GetNewClosure() + } +} + +AfterAll { + Remove-Variable -Name 'verifyPackageInvocations' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'verify-package-signatures.ps1' { + It 'verifies every package and symbol package found' { + Set-DotnetMock + + $output = Invoke-VerifyPackageSignatures + $output | Should -Match 'All 3 package signature\(s\) verified' + $global:verifyPackageInvocations.Count | Should -Be 3 + } + + It 'invokes dotnet nuget verify with --all' { + Set-DotnetMock + + Invoke-VerifyPackageSignatures | Out-Null + + $first = $global:verifyPackageInvocations | Select-Object -First 1 + ($first -join ' ') | Should -Match 'nuget verify --all' + } + + It 'fails when a package signature does not verify' { + Set-DotnetMock -FailPattern 'SqlServer' + + { Invoke-VerifyPackageSignatures } | Should -Throw '*Microsoft.SqlServer.Server.1.1.0.nupkg*' + } + + It 'checks every package before failing so all failures are reported' { + Set-DotnetMock -FailPattern '\.nupkg$' + + # Two of the three files are .nupkg; both must appear rather than only the first. + { Invoke-VerifyPackageSignatures } | Should -Throw '*failed for 2 of 3 package(s)*' + $global:verifyPackageInvocations.Count | Should -Be 3 + } + + It 'throws when no packages are found' { + Set-DotnetMock + $empty = Join-Path $TestDrive 'empty' + New-Item -ItemType Directory -Force -Path $empty | Out-Null + + { Invoke-VerifyPackageSignatures -PackagesPath $empty } | Should -Throw '*No package files were found*' + } +} diff --git a/eng/pipelines/onebranch/scripts/validate-localization.ps1 b/eng/pipelines/onebranch/scripts/validate-localization.ps1 new file mode 100644 index 0000000000..1922c78c5b --- /dev/null +++ b/eng/pipelines/onebranch/scripts/validate-localization.ps1 @@ -0,0 +1,180 @@ +<# +.SYNOPSIS + Validates localized Strings.*.resx files against Strings.resx. + +.PARAMETER ResourcesDirectory + Directory containing the English and localized Strings.resx files. + +.PARAMETER AllowlistPath + Optional JSON file containing approved English-value matches grouped by localized filename. +#> + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ResourcesDirectory, + + [string]$AllowlistPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-ResourceStrings { + param([Parameter(Mandatory)][string]$Path) + + $document = [System.Xml.Linq.XDocument]::Load($Path) + $strings = [System.Collections.Generic.Dictionary[string, string]]::new([System.StringComparer]::Ordinal) + foreach ($data in $document.Root.Elements('data')) { + $name = $data.Attribute('name') + $value = $data.Element('value') + if ($null -eq $name -or $null -eq $value) { + throw "Resource file '$Path' contains a element without a name or value." + } + + if ($strings.ContainsKey($name.Value)) { + throw "Resource file '$Path' contains duplicate key '$($name.Value)'." + } + + $strings.Add($name.Value, $value.Value) + } + + return $strings +} + +# Discover the neutral English resource and every culture-specific resource. The English file is +# the source of truth for the required key set and value-comparison checks below. +$resourcesPath = (Resolve-Path -LiteralPath $ResourcesDirectory).Path +$englishPath = Join-Path $resourcesPath 'Strings.resx' +if (-not (Test-Path -LiteralPath $englishPath -PathType Leaf)) { + throw "English resource file '$englishPath' was not found." +} + +$localizedFiles = @(Get-ChildItem -LiteralPath $resourcesPath -Filter 'Strings.*.resx' -File | Sort-Object Name) +if ($localizedFiles.Count -eq 0) { + throw "No localized Strings.*.resx files were found in '$resourcesPath'." +} + +$englishStrings = Get-ResourceStrings -Path $englishPath +$localizedFilesByName = [System.Collections.Generic.Dictionary[string, System.IO.FileInfo]]::new([System.StringComparer]::Ordinal) +foreach ($localizedFile in $localizedFiles) { + $localizedFilesByName.Add($localizedFile.Name, $localizedFile) +} + +# Load culture/key-specific exceptions for translations intentionally identical to English. The +# configuration is validated against the current resources so stale filenames and keys fail the +# build instead of silently weakening future validation. +$failures = [System.Collections.Generic.List[string]]::new() +$allowedEnglishMatches = [System.Collections.Generic.Dictionary[string, System.Collections.Generic.HashSet[string]]]::new([System.StringComparer]::Ordinal) +if (-not [string]::IsNullOrWhiteSpace($AllowlistPath)) { + if (-not (Test-Path -LiteralPath $AllowlistPath -PathType Leaf)) { + throw "Localization allowlist file '$AllowlistPath' was not found." + } + + $configuration = Get-Content -LiteralPath $AllowlistPath -Raw | ConvertFrom-Json + $englishValueMatchesProperty = $configuration.PSObject.Properties['AllowedEnglishValueMatches'] + if ($null -eq $englishValueMatchesProperty) { + throw "Localization allowlist file '$AllowlistPath' must define 'AllowedEnglishValueMatches'." + } + + foreach ($fileProperty in $englishValueMatchesProperty.Value.PSObject.Properties) { + if (-not $localizedFilesByName.ContainsKey($fileProperty.Name)) { + $failures.Add("Localization allowlist file references unknown resource file '$($fileProperty.Name)'.") + continue + } + + $keys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($key in @($fileProperty.Value)) { + if ([string]::IsNullOrWhiteSpace($key)) { + throw "Localization allowlist file contains an empty resource key for '$($fileProperty.Name)'." + } + if (-not $englishStrings.ContainsKey($key)) { + $failures.Add("Localization allowlist file references unknown resource key '$key' for '$($fileProperty.Name)'.") + continue + } + if ([string]::IsNullOrEmpty($englishStrings[$key])) { + throw "Localization allowlist key '$key' for '$($fileProperty.Name)' does not have a non-empty English value." + } + if (-not $keys.Add($key)) { + throw "Localization allowlist file contains duplicate key '$key' for '$($fileProperty.Name)'." + } + } + $allowedEnglishMatches.Add($fileProperty.Name, $keys) + } +} + +$allowedMatchCount = 0 +foreach ($localizedFile in $localizedFiles) { + $localizedStrings = Get-ResourceStrings -Path $localizedFile.FullName + + # Validate the key sets in both directions, then compare every non-empty English value with + # its localized value. Empty neutral values may remain empty because they contain no text to + # translate. + $missingOrEmptyKeys = [System.Collections.Generic.List[string]]::new() + $localizedOnlyKeys = [System.Collections.Generic.List[string]]::new() + $englishMatches = [System.Collections.Generic.List[string]]::new() + $staleAllowlistKeys = [System.Collections.Generic.List[string]]::new() + + foreach ($localizedKey in $localizedStrings.Keys) { + if (-not $englishStrings.ContainsKey($localizedKey)) { + $localizedOnlyKeys.Add($localizedKey) + } + } + + foreach ($entry in $englishStrings.GetEnumerator()) { + if (-not $localizedStrings.ContainsKey($entry.Key) -or + (-not [string]::IsNullOrEmpty($entry.Value) -and + [string]::IsNullOrWhiteSpace($localizedStrings[$entry.Key]))) { + $missingOrEmptyKeys.Add($entry.Key) + } + elseif (-not [string]::IsNullOrEmpty($entry.Value) -and + [System.StringComparer]::Ordinal.Equals($entry.Value, $localizedStrings[$entry.Key])) { + if ($allowedEnglishMatches.ContainsKey($localizedFile.Name) -and + $allowedEnglishMatches[$localizedFile.Name].Contains($entry.Key)) { + $allowedMatchCount++ + } + else { + $englishMatches.Add($entry.Key) + } + } + # An allowlist entry must be removed after its localized value changes; otherwise a future + # regression to the English value could be hidden by an obsolete exception. + elseif ($allowedEnglishMatches.ContainsKey($localizedFile.Name) -and + $allowedEnglishMatches[$localizedFile.Name].Contains($entry.Key)) { + $staleAllowlistKeys.Add($entry.Key) + } + } + + if ($missingOrEmptyKeys.Count -gt 0) { + $missingOrEmptyKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): missing keys or empty values: $($missingOrEmptyKeys -join ', ')") + } + if ($localizedOnlyKeys.Count -gt 0) { + $localizedOnlyKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): keys not found in Strings.resx: $($localizedOnlyKeys -join ', ')") + } + if ($englishMatches.Count -gt 0) { + $englishMatches.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): untranslated values match Strings.resx: $($englishMatches -join ', ')") + } + if ($staleAllowlistKeys.Count -gt 0) { + $staleAllowlistKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): allowlist entries no longer match Strings.resx: $($staleAllowlistKeys -join ', ')") + } +} + +if ($failures.Count -gt 0) { + foreach ($failure in $failures) { + Write-Host "##vso[task.logissue type=error]$failure" + } + $errorNoun = if ($failures.Count -eq 1) { 'error' } else { 'errors' } + throw "Localization validation failed with $($failures.Count) $errorNoun. Review the preceding errors." +} + +$fileNoun = if ($localizedFiles.Count -eq 1) { 'file' } else { 'files' } +Write-Host "Localization validation passed for $($localizedFiles.Count) localized $fileNoun. Resource keys checked: $($englishStrings.Count); approved English-value matches allowlisted: $allowedMatchCount." diff --git a/eng/pipelines/onebranch/scripts/validate-packages.ps1 b/eng/pipelines/onebranch/scripts/validate-packages.ps1 new file mode 100644 index 0000000000..af5149c9f5 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/validate-packages.ps1 @@ -0,0 +1,213 @@ +<# +.SYNOPSIS + Runs the PackageValidator tool over the NuGet packages produced by a OneBranch build. + +.DESCRIPTION + Invokes tools/PackageValidator once for a whole directory of packages rather than once per + package, because its most valuable checks are cross-package: every package in the SqlClient + family must carry the same version, and their inter-package dependency ranges must agree. + Validating one package at a time would silently skip all of those findings. + + The validator runs twice over the same inputs. The first run writes a machine-readable report + and never gates, so the report exists even when validation fails. The second run renders the + human-readable report and applies the gate, so a failing build shows its findings in its own + log rather than only in an artifact. + + Expected versions are supplied by the caller rather than derived here. The compute-versions + stage already computes every version the build stamps, and re-deriving them would reintroduce + the drift this validation exists to catch. + + Microsoft.SqlServer.Server is versioned separately from the SqlClient family, so its expected + versions are applied as a per-id override of the family wildcard. When it is not built in a + run, its package is absent from the drop and its expectations must be omitted entirely: the + validator rejects an expectation whose value is empty. + +.PARAMETER ValidatorPath + Path to the built PackageValidator.dll. Invoked through the managed assembly rather than the + native apphost so the same command works regardless of agent OS. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg files. Sibling .snupkg files must sit beside their + .nupkg for symbol matching to resolve, which is how the build jobs publish them. + +.PARAMETER ReportPath + Path of the JSON report to write. Parent directories are created as needed. + +.PARAMETER SqlClientPackageVersion + Package version expected of every package in the SqlClient family, applied as a wildcard. + Pointing every package at one value is what proves they agree, and also catches the case where + all of them are consistently wrong. + +.PARAMETER SqlClientFileVersion + Assembly file version expected of every assembly in the SqlClient family. + +.PARAMETER SqlServerPackageVersion + Package version expected of Microsoft.SqlServer.Server. Omit when SqlServer is not built. + +.PARAMETER SqlServerFileVersion + Assembly file version expected of Microsoft.SqlServer.Server. Omit when SqlServer is not built. + +.PARAMETER FailOn + Finding severities and/or categories that fail the build. Run the validator with --help to see + the available categories. Note that missing-symbols is a warning and package-unsigned is info, + so neither is covered by the error severity and both must be named explicitly. + + Accepts either an array or a single comma-separated string, because an Azure Pipelines task + argument line collapses to one token and PowerShell's -File mode does not split it. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing. + +.EXAMPLE + ./validate-packages.ps1 ` + -ValidatorPath ./PackageValidator.dll ` + -PackagesPath ./packages ` + -ReportPath ./out/report.json ` + -SqlClientPackageVersion 7.1.0-preview3.26238.3 ` + -SqlClientFileVersion 7.1.0.26238 ` + -FailOn error,missing-symbols + + Validates a family-only drop, failing on any error and on missing symbols. + +.NOTES + File Name : validate-packages.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : validate-packages-step.yml + + PackageValidator exit codes: + 0 - No gating findings. + 1 - The validator itself failed. + 2 - A --fail-on gate was tripped. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Path to the built PackageValidator.dll.")] + [ValidateNotNullOrEmpty()] + [string]$ValidatorPath, + + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for .nupkg files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(Mandatory = $true, HelpMessage = "Path of the JSON report to write.")] + [ValidateNotNullOrEmpty()] + [string]$ReportPath, + + [Parameter(Mandatory = $true, HelpMessage = "Package version expected of the SqlClient family.")] + [ValidateNotNullOrEmpty()] + [string]$SqlClientPackageVersion, + + [Parameter(Mandatory = $true, HelpMessage = "File version expected of the SqlClient family.")] + [ValidateNotNullOrEmpty()] + [string]$SqlClientFileVersion, + + [Parameter(HelpMessage = "Package version expected of Microsoft.SqlServer.Server, when built.")] + [string]$SqlServerPackageVersion = "", + + [Parameter(HelpMessage = "File version expected of Microsoft.SqlServer.Server, when built.")] + [string]$SqlServerFileVersion = "", + + [Parameter(Mandatory = $true, HelpMessage = "Severities and/or categories that fail the build.")] + [ValidateNotNullOrEmpty()] + [string[]]$FailOn, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Split on commas so a single "error,missing-symbols" token behaves like a two-element array. +$failOnTokens = @($FailOn -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +if ($failOnTokens.Count -eq 0) { + throw "FailOn must name at least one severity or category." +} + +Write-Host "=== Validate Packages Parameters ===" +Write-Host "ValidatorPath: ${ValidatorPath}" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "ReportPath: ${ReportPath}" +Write-Host "SqlClientPackageVersion: ${SqlClientPackageVersion}" +Write-Host "SqlClientFileVersion: ${SqlClientFileVersion}" +Write-Host "SqlServerPackageVersion: ${SqlServerPackageVersion}" +Write-Host "SqlServerFileVersion: ${SqlServerFileVersion}" +Write-Host "FailOn: $($failOnTokens -join ', ')" +Write-Host "====================================" + +if (-not (Test-Path -LiteralPath $ValidatorPath)) { + throw "PackageValidator was not found at '${ValidatorPath}'." +} + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Filter *.nupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No .nupkg files were found under '${PackagesPath}'." +} + +Write-Host "Validating $($packages.Count) package(s):" +$packages | ForEach-Object { Write-Host " $($_.Name)" } + +# A bare value applies to every package; an id=value pair overrides it for that package only. +$expectations = @( + "--expect-package-version", "*=${SqlClientPackageVersion}" + "--expect-file-version", "*=${SqlClientFileVersion}" +) + +# Both SqlServer versions travel together: supplying only one would assert half a package. +$hasSqlServerPackageVersion = -not [string]::IsNullOrWhiteSpace($SqlServerPackageVersion) +$hasSqlServerFileVersion = -not [string]::IsNullOrWhiteSpace($SqlServerFileVersion) +if ($hasSqlServerPackageVersion -ne $hasSqlServerFileVersion) { + throw "SqlServerPackageVersion and SqlServerFileVersion must be supplied together, or not at all." +} + +if ($hasSqlServerPackageVersion) { + $expectations += @( + "--expect-package-version", "Microsoft.SqlServer.Server=${SqlServerPackageVersion}" + "--expect-file-version", "Microsoft.SqlServer.Server=${SqlServerFileVersion}" + ) +} + +$gate = @() +foreach ($token in $failOnTokens) { + $gate += @("--fail-on", $token) +} + +Write-Host "Expectations: $($expectations -join ' ')" +Write-Host "Gate: $($gate -join ' ')" + +$reportDirectory = Split-Path -Parent $ReportPath +if ($reportDirectory) { + New-Item -ItemType Directory -Force -Path $reportDirectory | Out-Null +} + +# Reported before gating so the JSON exists even for a failing run. +& $DotnetPath $ValidatorPath $PackagesPath --json @expectations | + Set-Content -LiteralPath $ReportPath -Encoding utf8 +$reportExitCode = $LASTEXITCODE + +# This run is ungated, so any non-zero code means the validator itself failed and the report it +# produced cannot be trusted. Fail here rather than let the gated run obscure the real cause. +if ($reportExitCode -ne 0) { + throw "PackageValidator failed while writing the JSON report (exit code ${reportExitCode})." +} + +Write-Host "Wrote JSON report to ${ReportPath}" + +Write-Host "" +Write-Host "=== Package validation report ===" +& $DotnetPath $ValidatorPath $PackagesPath @expectations @gate +$exitCode = $LASTEXITCODE + +if ($exitCode -eq 0) { + Write-Host "" + Write-Host "Package validation passed." +} +elseif ($exitCode -eq 2) { + throw "Package validation failed: one or more findings matched the gate ($($failOnTokens -join ', '))." +} +else { + throw "PackageValidator exited unexpectedly with code ${exitCode}." +} diff --git a/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 b/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 new file mode 100644 index 0000000000..76c1495602 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 @@ -0,0 +1,99 @@ +<# +.SYNOPSIS + Verifies that every assembly shipped inside an official build's NuGet packages is Authenticode + signed. + +.DESCRIPTION + Expands each .nupkg beneath a directory and checks the Authenticode signature of every assembly + it contains, including native binaries under runtimes/. + + Packages are expanded rather than installed through NuGet so that every produced package is + covered without resolving dependencies, and so that the check does not depend on any single + package id. + + This complements PackageValidator, which reports strong-name state from assembly metadata + cross-platform. Authenticode verification requires the Windows trust store, so this script runs + only on Windows agents and only for official builds; non-official builds deliberately produce + unsigned assemblies. + + Every assembly is checked before failing, so a single run reports all unsigned assemblies + rather than stopping at the first. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg files to expand. + +.PARAMETER ExtractPath + Directory the packages are expanded into. Each package is expanded into its own subdirectory so + that identically-named assemblies from different packages cannot collide. Existing content for + a package is replaced. + +.EXAMPLE + ./verify-assembly-signatures.ps1 -PackagesPath ./packages -ExtractPath ./extract + + Expands every package beneath ./packages and verifies the signature of each assembly. + +.NOTES + File Name : verify-assembly-signatures.ps1 + Requires : PowerShell 7+ on Windows (Get-AuthenticodeSignature). + Called by : validate-packages-job.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for .nupkg files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(Mandatory = $true, HelpMessage = "Directory the packages are expanded into.")] + [ValidateNotNullOrEmpty()] + [string]$ExtractPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Write-Host "=== Verify Assembly Signatures Parameters ===" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "ExtractPath: ${ExtractPath}" +Write-Host "=============================================" + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Filter *.nupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No .nupkg files were found under '${PackagesPath}'." +} + +New-Item -ItemType Directory -Force -Path $ExtractPath | Out-Null + +Add-Type -AssemblyName System.IO.Compression.FileSystem +foreach ($package in $packages) { + $destination = Join-Path $ExtractPath $package.BaseName + if (Test-Path -LiteralPath $destination) { + Remove-Item -LiteralPath $destination -Recurse -Force + } + + Write-Host "Expanding $($package.Name)" + [System.IO.Compression.ZipFile]::ExtractToDirectory($package.FullName, $destination) +} + +$assemblies = @(Get-ChildItem -Path $ExtractPath -Recurse -File -Filter *.dll) +if ($assemblies.Count -eq 0) { + throw "No assemblies were found under '${ExtractPath}'." +} + +# Every assembly is checked before throwing so one run reports all failures. +$unsigned = @() +foreach ($signature in @(Get-AuthenticodeSignature -FilePath $assemblies.FullName)) { + if ($signature.Status -eq "Valid") { + Write-Host " OK $($signature.Path)" + } + else { + Write-Host " FAIL $($signature.Path) - $($signature.Status)" + $unsigned += $signature.Path + } +} + +if ($unsigned.Count -gt 0) { + throw "Authenticode verification failed for $($unsigned.Count) of $($assemblies.Count) assemblies." +} + +Write-Host "All $($assemblies.Count) assemblies are Authenticode signed." diff --git a/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 b/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 new file mode 100644 index 0000000000..74fa864c07 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 @@ -0,0 +1,72 @@ +<# +.SYNOPSIS + Verifies the NuGet signatures of every package produced by an official OneBranch build. + +.DESCRIPTION + Runs `dotnet nuget verify --all` over every .nupkg and .snupkg found beneath a directory, + confirming that each carries a valid, trusted signature. + + This complements PackageValidator, which reports signature *presence* from package metadata + cross-platform. Establishing that a signature is trusted requires the platform trust store, + which is why this runs separately and only on official builds. Non-official builds deliberately + produce unsigned packages, so verifying them would always fail. + + Every package is verified before failing, so a single run reports all unsigned packages rather + than stopping at the first. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg and .snupkg files. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing. + +.EXAMPLE + ./verify-package-signatures.ps1 -PackagesPath ./packages + + Verifies every package and symbol package beneath ./packages. + +.NOTES + File Name : verify-package-signatures.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : validate-packages-job.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for package files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Write-Host "=== Verify Package Signatures Parameters ===" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "============================================" + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Include *.nupkg, *.snupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No package files were found under '${PackagesPath}'." +} + +# Every package is checked before throwing so one run reports all failures. +$failed = @() +foreach ($package in $packages) { + Write-Host "Verifying $($package.Name)" + & $DotnetPath nuget verify --all $package.FullName + if ($LASTEXITCODE -ne 0) { + $failed += $package.Name + } +} + +if ($failed.Count -gt 0) { + throw "NuGet signature verification failed for $($failed.Count) of $($packages.Count) package(s): $($failed -join ', ')" +} + +Write-Host "All $($packages.Count) package signature(s) verified." diff --git a/eng/pipelines/onebranch/sqlclient-non-official.yml b/eng/pipelines/onebranch/sqlclient-non-official.yml index 2274d8cccf..88231ed15b 100644 --- a/eng/pipelines/onebranch/sqlclient-non-official.yml +++ b/eng/pipelines/onebranch/sqlclient-non-official.yml @@ -4,26 +4,30 @@ # See the LICENSE file in the project root for more information. # ################################################################################# +# The human-readable name applied to each run of this pipeline. name: $(Year:YY)$(DayOfYear)$(Rev:.r) -# No automated triggers; this pipeline must be run manually. +# No activity- or schedule-based triggers. This pipeline is run manually only. pr: none trigger: none # These parameters are visible in the Azure DevOps pipeline UI when a new run is queued. parameters: + + # When true, any SDL errors will break the build. When false, SDL errors will be logged but + # will not break the build. TSA bug filing is always disabled in the non-official pipeline + # (see the tsa block in globalSdl). + - name: breakOnSdlError + displayName: Break on SDL error + type: boolean + default: true + # True to enable debug information and steps. - name: debug displayName: Enable debug output type: boolean default: false - # True if this is a preview build. - - name: isPreview - displayName: Is this a preview build? - type: boolean - default: false - # True to publish symbols to private and public servers. - name: publishSymbols displayName: Publish symbols @@ -32,93 +36,43 @@ parameters: # Build parameters — select which packages to build. - # Build the Microsoft.SqlServer.Server package. - - name: buildSqlServerServer + # Build the Microsoft.SqlServer.Server package. The SqlClient family is always built; SqlServer + # is optional. When built, the SqlClient family depends on the freshly-built SqlServer package; + # when not built, the family depends on the most recently published SqlServer package. + - name: buildSqlServer displayName: Build Microsoft.SqlServer.Server type: boolean default: true - # Build Microsoft.Data.SqlClient and Extensions packages. - - name: buildSqlClient - displayName: Build Microsoft.Data.SqlClient and Extensions - type: boolean - default: true - - # Build the Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider package. - - name: buildAKVProvider - displayName: Build Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider - type: boolean - default: true - # Release parameters — select which packages to publish to NuGet. # All default to false; toggle at queue time for on-demand selective release. - # Release the Microsoft.SqlServer.Server package. - - name: releaseSqlServerServer + # Release the Microsoft.SqlServer.Server package (versioned separately). + - name: releaseSqlServer displayName: Release Microsoft.SqlServer.Server type: boolean default: false - # Release the Microsoft.Data.SqlClient.Internal.Logging package. - - name: releaseLogging - displayName: Release Microsoft.Data.SqlClient.Internal.Logging - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient.Extensions.Abstractions package. - - name: releaseAbstractions - displayName: Release Microsoft.Data.SqlClient.Extensions.Abstractions - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient package. + # Release the SqlClient family (Internal.Logging, Extensions.Abstractions, + # Microsoft.Data.SqlClient, Extensions.Azure, and the AlwaysEncrypted AzureKeyVaultProvider). + # The family is always released together at the shared SqlClient version. - name: releaseSqlClient - displayName: Release Microsoft.Data.SqlClient - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient.Extensions.Azure package. - - name: releaseAzure - displayName: Release Microsoft.Data.SqlClient.Extensions.Azure - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider package. - - name: releaseAKVProvider - displayName: Release Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider + displayName: Release SqlClient family type: boolean default: false variables: + - template: /eng/pipelines/common/variables/common-variables.yml@self - template: /eng/pipelines/onebranch/variables/onebranch-variables.yml@self + - template: /eng/pipelines/onebranch/variables/package-variables.yml@self - # Define the effective versions for all of the packages we build and release. - - ${{ if parameters.isPreview }}: - - name: effectiveSqlServerVersion - value: $(sqlServerPackagePreviewVersion) - - name: effectiveLoggingVersion - value: $(loggingPackagePreviewVersion) - - name: effectiveAbstractionsVersion - value: $(abstractionsPackagePreviewVersion) - - name: effectiveSqlClientVersion - value: $(mdsPackagePreviewVersion) - - name: effectiveAzureVersion - value: $(azurePackagePreviewVersion) - - name: effectiveAkvProviderVersion - value: $(akvPackagePreviewVersion) - - ${{ else }}: - - name: effectiveSqlServerVersion - value: $(sqlServerPackageVersion) - - name: effectiveLoggingVersion - value: $(loggingPackageVersion) - - name: effectiveAbstractionsVersion - value: $(abstractionsPackageVersion) - - name: effectiveSqlClientVersion - value: $(mdsPackageVersion) - - name: effectiveAzureVersion - value: $(azurePackageVersion) - - name: effectiveAkvProviderVersion - value: $(akvPackageVersion) + # Drives ContinuousIntegrationBuild=true in src/Directory.Build.props so this + # pipeline produces deterministic packages identical (modulo signing) to the + # official pipeline. Keeping this enabled here ensures the deterministic + # build path is exercised on every non-official run, catching regressions + # before they reach the official release pipeline. + - name: BuildForRelease + value: true resources: repositories: @@ -130,63 +84,255 @@ resources: extends: # See: https://aka.ms/obpipelines/templates template: /v2/OneBranch.NonOfficial.CrossPlat.yml@templates + parameters: - release: - # This indicates the pipeline category to deploy Box products. See: - # https://eng.ms/docs/products/onebranch/release/yamlreleasepipelines/deployboxproducts - category: NonAzure featureFlags: + # WindowsHostVersion selects the Windows *host VM* that our Windows build container runs on. + # This is a separate layer from the container image itself (WindowsContainerImage in + # onebranch-variables.yml): the host is the outer machine running the Docker engine, and the + # container is where our build steps actually execute. + # + # These two must be kept compatible. Windows containers can only run on a host whose OS + # version is compatible with the container's base image. A mismatch (e.g. a 2025 container on + # a 2022 host) fails to start the container unless Hyper-V isolation is forced. + WindowsHostVersion: + Version: 2025 + + # We do NOT set a LinuxHostVersion. Unlike Windows, Linux containers share the host kernel, + # so there is no host/container OS-version compatibility requirement to satisfy -- our Linux + # build image (LinuxContainerImage in onebranch-variables.yml) runs on the default OneBranch + # Linux host regardless of its distribution. + + # CDPx is OneBranch's predecessor build system. When EnableCDPxPAT is true (the OneBranch + # default), the governed templates inject a legacy CDPx Personal Access Token and its + # associated NuGet / Azure Artifacts authentication variables (CDP_DEFAULT_CLIENT_PAT, + # VSS_NUGET_ACCESSTOKEN, VSS_NUGET_URI_PREFIXES, etc.) into the build and Docker jobs so + # package restore against Azure DevOps feeds works without explicit auth. We don't rely on + # that legacy CDPx package-authentication path, so we disable it. EnableCDPxPAT: false - WindowsHostVersion: 1ESWindows2022 + + release: + # This indicates the pipeline category to deploy Box products. See: + # https://eng.ms/docs/products/onebranch/release/yamlreleasepipelines/deployboxproducts + category: NonAzure + # See: https://aka.ms/obpipelines/sdl + # + # The following SDL tasks are auto-injected by the OneBranch / 1ES Pipeline Templates and run + # WITHOUT any explicit globalSdl configuration. They were verified as running in recent + # pipeline runs, so we intentionally do not redeclare them here: + # + # - Component Governance / Component Detection / Secure Supply Chain + # Analysis (dependency vulnerability scanning) + # - AntiMalware Scanner (Binary + Source) + # - 1ES Secret Scanning (SPMI) + # - Generate SBoM Manifest + # - CodeQL 3000 (only run when it is explicitly opted in via codeql.compiled.enabled, and + # even then only on its ~72h cadence and on supported (non-PR/non-tag) + # branches.) + # globalSdl: - tsa: - # We disable TSA for non-official pipelines. This inhibits spurious TSA alerts and ADO - # task creation, since non-official builds are often done against development branches. - enabled: false + + # BREAK SEVERITY + # + # The SDL analyzer tasks never fail on findings; they only fail if the tool itself crashes or + # is misconfigured. The build break comes from the Post Analysis (Guardian Break) task, which + # reads the tool logs and fails when a finding meets or exceeds a minimum severity threshold. + # See https://aka.ms/gdn-azdo-break. + # + # Guardian normalises every finding to Error, Warning or Note. The threshold is cumulative, + # so a lower name is STRICTER, not looser: + # + # Error break on Error <-- Guardian's default + # Warning break on Error + Warning + # Note break on Error + Warning + Note + # + # Two knobs control it, in increasing order of precedence: + # + # globalSdl.severity threshold for every tool (maps to GdnBreakPolicyMinSev) + # globalSdl..severity per-tool override of the global threshold + # ob_sdl__severity per-job variable; overrides both of the above + # + # We omit `severity` everywhere and keep the Error-only default. OneBranch accepts ONLY + # Error, Warning or Note at this layer -- there is no explicit "Default" value to write, so + # inheriting the default requires omitting the key. Consequently, spelling out + # `severity: Error` on a tool is NOT equivalent to omitting it: it pins that tool to Error + # even if globalSdl.severity is later tightened. + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds + + # Snapshot of the SDL analyzer findings that pre-existed the breakOnSdlError rollout, so + # builds only break on NEW findings. Generated from the SDL analysis artifacts of a full + # non-official run and kept under .config/ alongside the other SDL tool configs + # (CredScan/PoliCheck/TSA). + baseline: + baselineFile: $(Build.SourcesDirectory)/.config/guardian/.gdnbaselines + + # Configure APIScan behaviour: + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds#apiscan + # + # https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-mohanb/security-integration/guardian-wiki/sdl-azdo-extension/apiscan-build-task#v2 + # apiscan: + # We want APIScan enabled by default for our jobs. However, some jobs don't produce any + # artifacts, and they will disable APIScan via the OneBranch per-job ob_sdl_apiscan_enabled + # variable. + enabled: true + + # APIScan errors break the build when breakOnSdlError is set. + break: ${{ parameters.breakOnSdlError }} + + # Use pre-release mode for non-official pipelines. + modeType: prerelease + + # The APIScan software name and version are NOT set here. Each package is registered with + # APIScan under its own name/version pair, so every build job sets ob_sdl_apiscan_softwareName + # and ob_sdl_apiscan_versionNumber for the package it builds (see build-buildproj-job.yml). + # Jobs that produce no assemblies disable APIScan instead, via ob_sdl_apiscan_enabled. + + # We want the standard level of logging. + verbosityLevel: standard + + armory: enabled: true - break: false - # Other ApiScan options are set by the jobs via ob_sdl_apiscan_* variables. + break: ${{ parameters.breakOnSdlError }} + + asyncSdl: + # Disabling this as it complicates the build process with minimal gain + enabled: false + + binskim: + enabled: true + break: ${{ parameters.breakOnSdlError }} + codeql: + # CodeQL 3000 is configured under the `compiled` key; `enabled` is a sub-property of + # `compiled` (not of `codeql`). CodeQL runs asynchronously on its own platform and cannot + # break the build by design, so it has no `break` value. compiled: + enabled: true + + credscan: enabled: true - sbom: - enabled: true - packageName: Microsoft.Data.SqlClient - packageVersion: $(mdsPackageVersion) + suppressionsFile: '$(REPO_ROOT)/.config/CredScanSuppressions.json' + + eslint: + # Only useful for repos with ECMAscript - which we do not have. + enabled: false + # Break value is wired preemptively so it takes effect if eslint is ever enabled. + break: ${{ parameters.breakOnSdlError }} + policheck: enabled: true - break: true - exclusionsFile: $(REPO_ROOT)\.config\PolicheckExclusions.xml - asyncSdl: + break: ${{ parameters.breakOnSdlError }} + exclusionsFile: '$(REPO_ROOT)\.config\PolicheckExclusions.xml' + + psscriptanalyzer: + # Static analysis of the repository's PowerShell scripts (e.g. under eng/). + enabled: true + break: ${{ parameters.breakOnSdlError }} + + roslyn: + # Enabling Roslyn SDL analysis here requires that our .NET builds _produce_ Roslyn findings. + # You will see this in the separate Roslyn build task. + # + # Note that the Roslyn-specific Guardian collector/sanitizer requires SARIF v1, so our + # analysis build deliberately emits v1. Other, generic Guardian tooling expects SARIF v2 and + # may log processing errors (for example, Post Analysis's SDL artifact report) even though + # Roslyn collection and Guardian policy ingestion succeed. + enabled: true + break: ${{ parameters.breakOnSdlError }} + + publishLogs: + enabled: true + + sbom: + enabled: true + # OneBranch resolves these from globalSdl only -- there is no per-job form -- so they + # indirect through variables that each build job sets to the package it produces. Jobs + # that publish no packages disable SBOM via ob_sdl_sbom_enabled rather than defaulting + # these. See build-buildproj-job.yml. + packageName: '$(sbomPackageName)' + packageVersion: '$(sbomPackageVersion)' + + tsa: + # TSA (Trust Services Automation) files SDL analysis findings as Azure DevOps bug work + # items. It is hardcoded DISABLED in the non-official pipeline. + # + # Why disabled here: + # The non-official pipeline is a manual-only, developer-helper build used to validate + # SDL scans and iterate on fixes before they land. We do not want it filing (or + # updating) bug work items -- that would create churn and duplicate the bugs owned by + # the official pipeline, where TSA is hardcoded enabled. Here, findings surface + # directly in the build instead, gated by breakOnSdlError. + # + # Interaction with other SDL tool defaults: + # OneBranch derives each SDL tool's DEFAULT `break` value from whether TSA is enabled -- + # per the OneBranch "Customize SDL" docs, most tools default to `break: false` when TSA + # is enabled and `break: true` when it is disabled. We do NOT rely on that implicit + # coupling: every tool above sets `break: ${{ parameters.breakOnSdlError }}` explicitly, + # which overrides the TSA-derived default. We have hit bugs in OneBranch's implicit + # break-altering logic in the past, so both `tsa.enabled` and each tool's `break` are + # set explicitly to keep the behaviour deterministic. + # + # Otherwise independent of breakOnSdlError: + # Aside from that default-flipping, TSA and breakOnSdlError are orthogonal. Disabling + # TSA here does not by itself force breaking -- breakOnSdlError is what controls whether + # findings fail the build. enabled: false - credscan: - enabled: true - suppressionsFile: $(REPO_ROOT)/.config/CredScanSuppressions.json - binskim: - enabled: true - armory: - enabled: true - break: true - eslint: - enabled: false - roslyn: - enabled: true - break: true - publishLogs: - enabled: true - tsaOptionsPath: $(REPO_ROOT)\.config\tsaoptions.json - disableLegacyManifest: true + # Keep this in sync with Official even though TSA is disabled here. + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + stages: + # Compile-time guard (1ES/OneBranch ': error' idiom): releasing Microsoft.SqlServer.Server + # requires building it this run, otherwise there is no package to release. The invalid item + # is only emitted for the bad parameter combination and fails template expansion with the + # message below. + - ${{ if and(eq(parameters.releaseSqlServer, true), eq(parameters.buildSqlServer, false)) }}: + - 'Invalid parameters: releaseSqlServer=true requires buildSqlServer=true. There is no freshly-built Microsoft.SqlServer.Server package to release when buildSqlServer is false.': error + + - template: /eng/pipelines/onebranch/stages/compute-versions-stage.yml@self + parameters: + buildSqlServer: ${{ parameters.buildSqlServer }} + - template: /eng/pipelines/onebranch/stages/build-stages.yml@self parameters: - debug: ${{ parameters.debug }} - isPreview: ${{ parameters.isPreview }} + isOfficial: false # This is a non-official pipeline. + buildSqlServer: ${{ parameters.buildSqlServer }} + + abstractionsArtifactsName: '${{ variables.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ variables.akvProviderArtifactsName }}' + azureArtifactsName: '${{ variables.azureArtifactsName }}' + loggingArtifactsName: '${{ variables.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ variables.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ variables.sqlServerArtifactsName }}' + + signingAppRegistrationClientId: '$(SigningAppRegistrationClientId)' + signingAppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + signingAuthAkvName: '$(SigningAuthAkvName)' + signingAuthSignCertName: '$(SigningAuthSignCertName)' + signingEsrpClientId: '$(SigningEsrpClientId)' + signingEsrpConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + + - template: /eng/pipelines/onebranch/stages/publish-symbols-stage.yml@self + parameters: publishSymbols: ${{ parameters.publishSymbols }} - buildSqlServerServer: ${{ parameters.buildSqlServerServer }} - buildSqlClient: ${{ parameters.buildSqlClient }} - buildAKVProvider: ${{ parameters.buildAKVProvider }} + buildSqlServer: ${{ parameters.buildSqlServer }} + + abstractionsArtifactsName: '${{ variables.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ variables.akvProviderArtifactsName }}' + azureArtifactsName: '${{ variables.azureArtifactsName }}' + loggingArtifactsName: '${{ variables.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ variables.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ variables.sqlServerArtifactsName }}' + + symbolsAzureSubscription: '$(SymbolsAzureSubscription)' + symbolsPublishProjectName: '$(SymbolsPublishProjectNameSqlClient)' + # Non-Official pipelines must publish to the PPE symbol server. + symbolsPublishServer: '$(SymbolsPublishServerPpe)' + symbolsPublishTokenUri: '$(SymbolsPublishTokenUriPpe)' + symbolsUploadAccount: '$(SymbolsUploadAccount)' - template: /eng/pipelines/onebranch/stages/release-stages.yml@self parameters: @@ -196,9 +342,15 @@ extends: # This is _not_ an official pipeline. isOfficial: false stageNameSuffix: test - releaseSqlServerServer: ${{ parameters.releaseSqlServerServer }} - releaseLogging: ${{ parameters.releaseLogging }} - releaseAbstractions: ${{ parameters.releaseAbstractions }} + + publishSymbols: ${{ parameters.publishSymbols }} + + abstractionsArtifactsName: '${{ variables.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ variables.akvProviderArtifactsName }}' + azureArtifactsName: '${{ variables.azureArtifactsName }}' + loggingArtifactsName: '${{ variables.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ variables.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ variables.sqlServerArtifactsName }}' + + releaseSqlServer: ${{ parameters.releaseSqlServer }} releaseSqlClient: ${{ parameters.releaseSqlClient }} - releaseAzure: ${{ parameters.releaseAzure }} - releaseAKVProvider: ${{ parameters.releaseAKVProvider }} diff --git a/eng/pipelines/onebranch/sqlclient-official.yml b/eng/pipelines/onebranch/sqlclient-official.yml index c9548b25d6..b8521fc0d7 100644 --- a/eng/pipelines/onebranch/sqlclient-official.yml +++ b/eng/pipelines/onebranch/sqlclient-official.yml @@ -4,44 +4,36 @@ # See the LICENSE file in the project root for more information. # ################################################################################# +# The human-readable name applied to each run of this pipeline. name: $(Year:YY)$(DayOfYear)$(Rev:.r) -# No PR-based triggers. +# No activity-based triggers. pr: none - -# We trigger runs on activity on the internal/main branch, but not on any other branches. This -# pipeline is intended to only run against the ADO.Net dotnet-sqlclient repo. -trigger: - branches: - include: - - internal/main +trigger: none # We run this pipeline on a daily schedule. schedules: - - cron: "30 4 * * *" - displayName: Daily 04:30 UTC Build + - cron: "0 23 * * *" + displayName: 7.1 Daily Official Build (23:00 UTC) branches: include: - - internal/main + - internal/release/7.1 always: true # These parameters are visible in the Azure DevOps pipeline UI when a new run is queued. parameters: - # True to enable debug information and steps. - - name: debug - displayName: Enable debug output - type: boolean - default: false - # Push packages to NuGet Production (otherwise pushes to NuGet Test). - - name: releaseToProduction - displayName: Release to NuGet Production + # When true, any SDL errors will break the build. When false, SDL errors will be logged but + # will not break the build. TSA bug filing is always enabled in the official pipeline and is + # independent of this parameter (see the tsa block in globalSdl). + - name: breakOnSdlError + displayName: Break on SDL error type: boolean - default: false + default: true - # True if this is a preview build. - - name: isPreview - displayName: Is this a preview build? + # True to enable debug information and steps. + - name: debug + displayName: Enable debug output type: boolean default: false @@ -51,95 +43,51 @@ parameters: type: boolean default: false - # Build parameters — select which packages to build. - - # Build the Microsoft.SqlServer.Server package. - - name: buildSqlServerServer - displayName: Build Microsoft.SqlServer.Server + # When true, publish symbols and push NuGet packages to Production environments. When false, + # symbols use PPE and NuGet packages use QA/Test. + - name: releaseToProduction + displayName: Publish Symbols and NuGet Packages to Production type: boolean - default: true + default: false - # Build Microsoft.Data.SqlClient and Extensions packages. - - name: buildSqlClient - displayName: Build Microsoft.Data.SqlClient and Extensions - type: boolean - default: true + # Build parameters — select which packages to build. - # Build the Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider package. - - name: buildAKVProvider - displayName: Build Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider + # Build the Microsoft.SqlServer.Server package. The SqlClient family is always built; SqlServer + # is optional. When built, the SqlClient family depends on the freshly-built SqlServer package; + # when not built, the family depends on the most recently published SqlServer package. + - name: buildSqlServer + displayName: Build Microsoft.SqlServer.Server type: boolean - default: true + default: false # Release parameters — select which packages to publish to NuGet. # All default to false; toggle at queue time for on-demand selective release. - # Release the Microsoft.SqlServer.Server package. - - name: releaseSqlServerServer + # Release the Microsoft.SqlServer.Server package (versioned separately). + - name: releaseSqlServer displayName: Release Microsoft.SqlServer.Server type: boolean default: false - # Release the Microsoft.Data.SqlClient.Internal.Logging package. - - name: releaseLogging - displayName: Release Microsoft.Data.SqlClient.Internal.Logging - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient.Extensions.Abstractions package. - - name: releaseAbstractions - displayName: Release Microsoft.Data.SqlClient.Extensions.Abstractions - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient package. + # Release the SqlClient family (Internal.Logging, Extensions.Abstractions, + # Microsoft.Data.SqlClient, Extensions.Azure, and the AlwaysEncrypted AzureKeyVaultProvider). + # The family is always released together at the shared SqlClient version. - name: releaseSqlClient - displayName: Release Microsoft.Data.SqlClient - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient.Extensions.Azure package. - - name: releaseAzure - displayName: Release Microsoft.Data.SqlClient.Extensions.Azure - type: boolean - default: false - - # Release the Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider package. - - name: releaseAKVProvider - displayName: Release Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider + displayName: Release SqlClient family type: boolean default: false variables: + - template: /eng/pipelines/common/variables/common-variables.yml@self - template: /eng/pipelines/onebranch/variables/onebranch-variables.yml@self + - template: /eng/pipelines/onebranch/variables/package-variables.yml@self - # Define the effective versions for all of the packages we build and release. - - ${{ if parameters.isPreview }}: - - name: effectiveSqlServerVersion - value: $(sqlServerPackagePreviewVersion) - - name: effectiveLoggingVersion - value: $(loggingPackagePreviewVersion) - - name: effectiveAbstractionsVersion - value: $(abstractionsPackagePreviewVersion) - - name: effectiveSqlClientVersion - value: $(mdsPackagePreviewVersion) - - name: effectiveAzureVersion - value: $(azurePackagePreviewVersion) - - name: effectiveAkvProviderVersion - value: $(akvPackagePreviewVersion) - - ${{ else }}: - - name: effectiveSqlServerVersion - value: $(sqlServerPackageVersion) - - name: effectiveLoggingVersion - value: $(loggingPackageVersion) - - name: effectiveAbstractionsVersion - value: $(abstractionsPackageVersion) - - name: effectiveSqlClientVersion - value: $(mdsPackageVersion) - - name: effectiveAzureVersion - value: $(azurePackageVersion) - - name: effectiveAkvProviderVersion - value: $(akvPackageVersion) + # Drives ContinuousIntegrationBuild=true in src/Directory.Build.props so the shipped + # DLLs/PDBs embed portable /_/... source paths and the produced NuGet packages pass + # NuGet Package Explorer's "Deterministic (build)" check. Set on both OneBranch + # pipelines; PR/CI builds leave it unset (false). + - name: BuildForRelease + value: true resources: repositories: @@ -150,68 +98,256 @@ resources: extends: # See: https://aka.ms/obpipelines/templates - template: /v2/OneBranch.Official.CrossPlat.yml@templates + template: '/v2/OneBranch.Official.CrossPlat.yml@templates' + parameters: - release: - # This indicates the pipeline category to deploy Box products. See: - # https://eng.ms/docs/products/onebranch/release/yamlreleasepipelines/deployboxproducts - category: NonAzure featureFlags: + # WindowsHostVersion selects the Windows *host VM* that our Windows build container runs on. + # This is a separate layer from the container image itself (WindowsContainerImage in + # onebranch-variables.yml): the host is the outer machine running the Docker engine, and the + # container is where our build steps actually execute. + # + # These two must be kept compatible. Windows containers can only run on a host whose OS + # version is compatible with the container's base image. A mismatch (e.g. a 2025 container on + # a 2022 host) fails to start the container unless Hyper-V isolation is forced. + WindowsHostVersion: + Version: 2025 + + # We do NOT set a LinuxHostVersion. Unlike Windows, Linux containers share the host kernel, + # so there is no host/container OS-version compatibility requirement to satisfy -- our Linux + # build image (LinuxContainerImage in onebranch-variables.yml) runs on the default OneBranch + # Linux host regardless of its distribution. + + # CDPx is OneBranch's predecessor build system. When EnableCDPxPAT is true (the OneBranch + # default), the governed templates inject a legacy CDPx Personal Access Token and its + # associated NuGet / Azure Artifacts authentication variables (CDP_DEFAULT_CLIENT_PAT, + # VSS_NUGET_ACCESSTOKEN, VSS_NUGET_URI_PREFIXES, etc.) into the build and Docker jobs so + # package restore against Azure DevOps feeds works without explicit auth. We don't rely on + # that legacy CDPx package-authentication path, so we disable it. EnableCDPxPAT: false - WindowsHostVersion: 1ESWindows2022 + + release: + # This indicates the pipeline category to deploy Box products. See: + # https://eng.ms/docs/products/onebranch/release/yamlreleasepipelines/deployboxproducts + category: NonAzure + # See: https://aka.ms/obpipelines/sdl + # + # The following SDL tasks are auto-injected by the OneBranch / 1ES Pipeline Templates and run + # WITHOUT any explicit globalSdl configuration. They were verified as running in recent + # pipeline runs, so we intentionally do not redeclare them here: + # + # - Component Governance / Component Detection / Secure Supply Chain + # Analysis (dependency vulnerability scanning) + # - AntiMalware Scanner (Binary + Source) + # - 1ES Secret Scanning (SPMI) + # - Generate SBoM Manifest + # - CodeSign Validation (runs on the signing path; nothing is signed in non-official runs) + # - CodeQL 3000 (ALWAYS enabled via the 1ESPT (Official.CrossPlat) entry point, regardless of + # the codeql config below.) + # globalSdl: - tsa: - # The OneBranch template will set 'break' to false for the other SDL tools when TSA is - # enabled. This allows TSA to gather the results and publish them for downstream analysis. - enabled: true + + # BREAK SEVERITY + # + # The SDL analyzer tasks never fail on findings; they only fail if the tool itself crashes or + # is misconfigured. The build break comes from the Post Analysis (Guardian Break) task, which + # reads the tool logs and fails when a finding meets or exceeds a minimum severity threshold. + # See https://aka.ms/gdn-azdo-break. + # + # Guardian normalises every finding to Error, Warning or Note. The threshold is cumulative, + # so a lower name is STRICTER, not looser: + # + # Error break on Error <-- Guardian's default + # Warning break on Error + Warning + # Note break on Error + Warning + Note + # + # Two knobs control it, in increasing order of precedence: + # + # globalSdl.severity threshold for every tool (maps to GdnBreakPolicyMinSev) + # globalSdl..severity per-tool override of the global threshold + # ob_sdl__severity per-job variable; overrides both of the above + # + # We omit `severity` everywhere and keep the Error-only default. OneBranch accepts ONLY + # Error, Warning or Note at this layer -- there is no explicit "Default" value to write, so + # inheriting the default requires omitting the key. Consequently, spelling out + # `severity: Error` on a tool is NOT equivalent to omitting it: it pins that tool to Error + # even if globalSdl.severity is later tightened. + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds + + # Snapshot of the SDL analyzer findings that pre-existed the breakOnSdlError rollout, so + # builds only break on NEW findings. Generated from the SDL analysis artifacts of a full + # non-official run and kept under .config/ alongside the other SDL tool configs + # (CredScan/PoliCheck/TSA). + baseline: + baselineFile: $(Build.SourcesDirectory)/.config/guardian/.gdnbaselines + + # Configure APIScan behaviour: + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds#apiscan + # + # https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-mohanb/security-integration/guardian-wiki/sdl-azdo-extension/apiscan-build-task#v2 + # apiscan: + # We want APIScan enabled by default for our jobs. However, some jobs don't produce any + # artifacts, and they will disable APIScan via the OneBranch per-job ob_sdl_apiscan_enabled + # variable. enabled: true - # TODO(https://sqlclientdrivers.visualstudio.com/ADO.Net/_workitems/edit/42858): - # We have temporarily disabled breaking the build on ApiScan results until we can: - # - Register our new packages with API Scan, and/or - # - Publish MSDN/Learn/etc documentation for the new packages. - break: false - # Other ApiScan options are set by the jobs via ob_sdl_apiscan_* variables. + + # APIScan errors break the build when breakOnSdlError is set. + break: ${{ parameters.breakOnSdlError }} + + # Use release mode for official pipelines. + modeType: release + + # The APIScan software name and version are NOT set here. Each package is registered with + # APIScan under its own name/version pair, so every build job sets ob_sdl_apiscan_softwareName + # and ob_sdl_apiscan_versionNumber for the package it builds (see build-buildproj-job.yml). + # Jobs that produce no assemblies disable APIScan instead, via ob_sdl_apiscan_enabled. + + # We want the standard level of logging. + verbosityLevel: standard + + armory: + enabled: true + break: ${{ parameters.breakOnSdlError }} + + asyncSdl: + # Disabling this as it complicates the build process with minimal gain + enabled: false + + binskim: + enabled: true + break: ${{ parameters.breakOnSdlError }} + codeql: + # CodeQL 3000 is configured under the `compiled` key; `enabled` is a sub-property of + # `compiled` (not of `codeql`). CodeQL runs asynchronously on its own platform and cannot + # break the build by design, so it has no `break` value. compiled: + enabled: true + + credscan: enabled: true + suppressionsFile: '$(REPO_ROOT)/.config/CredScanSuppressions.json' + + eslint: + # Only useful for repos with ECMAscript - which we do not have. + enabled: false + # Break value is wired preemptively so it takes effect if eslint is ever enabled. + break: ${{ parameters.breakOnSdlError }} + + policheck: + enabled: true + break: ${{ parameters.breakOnSdlError }} + exclusionsFile: '$(REPO_ROOT)\.config\PolicheckExclusions.xml' + + psscriptanalyzer: + # Static analysis of the repository's PowerShell scripts (e.g. under eng/). + enabled: true + break: ${{ parameters.breakOnSdlError }} + + roslyn: + # Enabling Roslyn SDL analysis here requires that our .NET builds _produce_ Roslyn findings. + # You will see this in the separate Roslyn build task. + # + # Note that the Roslyn-specific Guardian collector/sanitizer requires SARIF v1, so our + # analysis build deliberately emits v1. Other, generic Guardian tooling expects SARIF v2 and + # may log processing errors (for example, Post Analysis's SDL artifact report) even though + # Roslyn collection and Guardian policy ingestion succeed. + enabled: true + break: ${{ parameters.breakOnSdlError }} + + publishLogs: + enabled: true + sbom: enabled: true - packageName: Microsoft.Data.SqlClient - packageVersion: $(mdsPackageVersion) - policheck: + # OneBranch resolves these from globalSdl only -- there is no per-job form -- so they + # indirect through variables that each build job sets to the package it produces. Jobs + # that publish no packages disable SBOM via ob_sdl_sbom_enabled rather than defaulting + # these. See build-buildproj-job.yml. + packageName: '$(sbomPackageName)' + packageVersion: '$(sbomPackageVersion)' + + tsa: + # TSA (Trust Services Automation) files SDL analysis findings as Azure DevOps bug work + # items. It is hardcoded enabled in the official pipeline so that every SDL finding is + # always tracked as a bug, regardless of how any individual run terminates. + # + # Interaction with other SDL tool defaults: + # OneBranch derives each SDL tool's DEFAULT `break` value from whether TSA is enabled -- + # per the OneBranch "Customize SDL" docs, most tools default to `break: false` when TSA + # is enabled and `break: true` when it is disabled. We do NOT rely on that implicit + # coupling: every tool above sets `break: ${{ parameters.breakOnSdlError }}` explicitly, + # which overrides the TSA-derived default. We have hit bugs in OneBranch's implicit + # break-altering logic in the past, so both `tsa.enabled` and each tool's `break` are + # set explicitly to keep the behaviour deterministic. + # + # Otherwise independent of breakOnSdlError: + # Aside from that default-flipping, TSA and breakOnSdlError are orthogonal. TSA files + # bugs from the complete finding set (the TSAUpload step runs after all analyzers have + # produced results), while breakOnSdlError separately controls whether those same + # findings also fail the build. Enabling TSA here does not suppress breaking, and + # breaking does not prevent bug filing within a job. enabled: true - break: true - exclusionsFile: $(REPO_ROOT)\.config\PolicheckExclusions.xml - asyncSdl: - enabled: false - credscan: - enabled: true - suppressionsFile: $(REPO_ROOT)/.config/CredScanSuppressions.json - binskim: - enabled: true - armory: - enabled: true - break: true - eslint: - enabled: false - roslyn: - enabled: true - break: true - publishLogs: - enabled: true - tsaOptionsPath: $(REPO_ROOT)\.config\tsaoptions.json - disableLegacyManifest: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + stages: + # Compile-time guard (1ES/OneBranch ': error' idiom): releasing Microsoft.SqlServer.Server + # requires building it this run, otherwise there is no package to release. The invalid item + # is only emitted for the bad parameter combination and fails template expansion with the + # message below. + - ${{ if and(eq(parameters.releaseSqlServer, true), eq(parameters.buildSqlServer, false)) }}: + - 'Invalid parameters: releaseSqlServer=true requires buildSqlServer=true. There is no freshly-built Microsoft.SqlServer.Server package to release when buildSqlServer is false.': error + + - template: /eng/pipelines/onebranch/stages/compute-versions-stage.yml@self + parameters: + buildSqlServer: ${{ parameters.buildSqlServer }} + - template: /eng/pipelines/onebranch/stages/build-stages.yml@self parameters: - debug: ${{ parameters.debug }} - isPreview: ${{ parameters.isPreview }} + isOfficial: true # This is an official pipeline. + buildSqlServer: ${{ parameters.buildSqlServer }} + + abstractionsArtifactsName: '${{ variables.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ variables.akvProviderArtifactsName }}' + azureArtifactsName: '${{ variables.azureArtifactsName }}' + loggingArtifactsName: '${{ variables.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ variables.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ variables.sqlServerArtifactsName }}' + + signingAppRegistrationClientId: '$(SigningAppRegistrationClientId)' + signingAppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + signingAuthAkvName: '$(SigningAuthAkvName)' + signingAuthSignCertName: '$(SigningAuthSignCertName)' + signingEsrpClientId: '$(SigningEsrpClientId)' + signingEsrpConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + + - template: /eng/pipelines/onebranch/stages/publish-symbols-stage.yml@self + parameters: publishSymbols: ${{ parameters.publishSymbols }} - buildSqlServerServer: ${{ parameters.buildSqlServerServer }} - buildSqlClient: ${{ parameters.buildSqlClient }} - buildAKVProvider: ${{ parameters.buildAKVProvider }} + buildSqlServer: ${{ parameters.buildSqlServer }} + + abstractionsArtifactsName: '${{ variables.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ variables.akvProviderArtifactsName }}' + azureArtifactsName: '${{ variables.azureArtifactsName }}' + loggingArtifactsName: '${{ variables.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ variables.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ variables.sqlServerArtifactsName }}' + + symbolsAzureSubscription: '$(SymbolsAzureSubscription)' + symbolsPublishProjectName: '$(SymbolsPublishProjectNameSqlClient)' + # Symbol server target follows releaseToProduction: Production for + # real releases, PPE for test/QA releases. + ${{ if eq(parameters.releaseToProduction, true) }}: + symbolsPublishServer: '$(SymbolsPublishServerProd)' + symbolsPublishTokenUri: '$(SymbolsPublishTokenUriProd)' + ${{ else }}: + symbolsPublishServer: '$(SymbolsPublishServerPPE)' + symbolsPublishTokenUri: '$(SymbolsPublishTokenUriPPE)' + symbolsUploadAccount: '$(SymbolsUploadAccount)' - template: /eng/pipelines/onebranch/stages/release-stages.yml@self parameters: @@ -220,9 +356,15 @@ extends: # This is an official pipeline. isOfficial: true stageNameSuffix: production - releaseSqlServerServer: ${{ parameters.releaseSqlServerServer }} - releaseLogging: ${{ parameters.releaseLogging }} - releaseAbstractions: ${{ parameters.releaseAbstractions }} + + publishSymbols: ${{ parameters.publishSymbols }} + + abstractionsArtifactsName: '${{ variables.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ variables.akvProviderArtifactsName }}' + azureArtifactsName: '${{ variables.azureArtifactsName }}' + loggingArtifactsName: '${{ variables.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ variables.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ variables.sqlServerArtifactsName }}' + + releaseSqlServer: ${{ parameters.releaseSqlServer }} releaseSqlClient: ${{ parameters.releaseSqlClient }} - releaseAzure: ${{ parameters.releaseAzure }} - releaseAKVProvider: ${{ parameters.releaseAKVProvider }} diff --git a/eng/pipelines/onebranch/stages/build-stages.yml b/eng/pipelines/onebranch/stages/build-stages.yml index 0b6962f1af..34f88bd9b3 100644 --- a/eng/pipelines/onebranch/stages/build-stages.yml +++ b/eng/pipelines/onebranch/stages/build-stages.yml @@ -12,43 +12,63 @@ # - build_independent: builds packages with no cross-package dependencies (Logging, SqlServer) # - build_abstractions: builds the Abstractions package, which depends on Logging # - build_dependent: builds packages with dependencies on Abstractions (SqlClient, Azure) -# - build_addons: builds add-on packages with dependencies on the core packages (AKV Provider) -# -# This template depends on the following runtime (i.e. macro expansion) variables being defined: -# -# - effectiveSqlServerVersion -# - effectiveLoggingVersion -# - effectiveAbstractionsVersion -# - effectiveSqlClientVersion -# - effectiveAzureVersion -# - effectiveAkvProviderVersion +# - build_addons: builds add-on packages with dependencies on the core packages (Akv Provider) parameters: # ── General parameters ───────────────────────────────────────────────── - # True to enable debug information and steps. - - name: debug + # True if this is an official build, which runs additional ESRP malware scanning + # and codesigning steps. + - name: isOfficial type: boolean - # True if this is a preview build, which uses the preview version numbers from - # common-variables.yml. - - name: isPreview - type: boolean + # ── Build parameters ─────────────────────────────────────────────────── - # True to publish symbols to public and private servers. - - name: publishSymbols + # Whether to build Microsoft.SqlServer.Server. The SqlClient family is always built; SqlServer + # is optional. When built, the SqlClient family depends on the freshly-built SqlServer package; + # when not built, the family depends on the most recently published SqlServer package. + - name: buildSqlServer type: boolean - # ── Build parameters ─────────────────────────────────────────────────── + # Package Parameters ----------------------------------------------------- - - name: buildSqlServerServer - type: boolean + - name: abstractionsArtifactsName + type: string - - name: buildSqlClient - type: boolean + - name: akvProviderArtifactsName + type: string - - name: buildAKVProvider - type: boolean + - name: azureArtifactsName + type: string + + - name: loggingArtifactsName + type: string + + - name: sqlClientArtifactsName + type: string + + - name: sqlServerArtifactsName + type: string + + # Signing Parameters ----------------------------------------------------- + + - name: signingAppRegistrationClientId + type: string + + - name: signingAppRegistrationTenantId + type: string + + - name: signingAuthAkvName + type: string + + - name: signingAuthSignCertName + type: string + + - name: signingEsrpClientId + type: string + + - name: signingEsrpConnectedServiceName + type: string stages: # ==================================================================== @@ -57,164 +77,298 @@ stages: # ==================================================================== - stage: build_independent displayName: "Build Independent Packages" + dependsOn: compute_versions + + variables: + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlServerFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerFileVersion'] ] + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] + - name: sqlServerApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerApiScanVersion'] ] jobs: - - ${{ if or(eq(parameters.buildAKVProvider, true), eq(parameters.buildSqlClient, true)) }}: - - template: /eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml@self - parameters: - packageName: Logging - packageFullName: Microsoft.Data.SqlClient.Internal.Logging - packageVersion: $(effectiveLoggingVersion) - versionProperties: >- - -p:LoggingPackageVersion=$(effectiveLoggingVersion) - -p:LoggingAssemblyFileVersion=$(loggingAssemblyFileVersion) - assemblyFileVersion: $(loggingAssemblyFileVersion) - publishSymbols: ${{ parameters.publishSymbols }} - esrpConnectedServiceName: $(ESRPConnectedServiceName) - esrpClientId: $(ESRPClientId) - appRegistrationClientId: $(AppRegistrationClientId) - appRegistrationTenantId: $(AppRegistrationTenantId) - authAkvName: $(AuthAKVName) - authSignCertName: $(AuthSignCertName) - - - ${{ if eq(parameters.buildSqlServerServer, true) }}: - - template: /eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml@self + # Build Microsoft.Data.SqlClient.Internal.Logging + - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self + parameters: + apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Internal.Logging/dlls' + apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Internal.Logging/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' + shouldSignPackage: ${{ parameters.isOfficial }} + signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + signingAuthAkvName: '${{ parameters.signingAuthAkvName }}' + signingAuthSignCertName: '${{ parameters.signingAuthSignCertName }}' + signingEsrpClientId: '${{ parameters.signingEsrpClientId }}' + signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + + dependencies: [] + fileVersion: '$(sqlClientFileVersion)' + packageFullName: 'Microsoft.Data.SqlClient.Internal.Logging' + packageShortName: 'Logging' + versionPropertySuffix: 'SqlClient' + packageVersion: '$(sqlClientPackageVersion)' + + + - ${{ if eq(parameters.buildSqlServer, true) }}: + # Build Microsoft.SqlServer.Server + - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self parameters: - packageName: SqlServer - packageFullName: Microsoft.SqlServer.Server - packageVersion: $(effectiveSqlServerVersion) - versionProperties: >- - -p:SqlServerAssemblyFileVersion=$(sqlServerAssemblyFileVersion) - -p:SqlServerPackageVersion=$(effectiveSqlServerVersion) - assemblyFileVersion: $(sqlServerAssemblyFileVersion) - publishSymbols: ${{ parameters.publishSymbols }} - esrpConnectedServiceName: $(ESRPConnectedServiceName) - esrpClientId: $(ESRPClientId) - appRegistrationClientId: $(AppRegistrationClientId) - appRegistrationTenantId: $(AppRegistrationTenantId) - authAkvName: $(AuthAKVName) - authSignCertName: $(AuthSignCertName) + apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.SqlServer.Server/dlls' + apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.SqlServer.Server/pdbs' + apiScanSoftwareVersion: '$(sqlServerApiScanVersion)' + shouldSignPackage: ${{ parameters.isOfficial }} + signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + signingAuthAkvName: '${{ parameters.signingAuthAkvName }}' + signingAuthSignCertName: '${{ parameters.signingAuthSignCertName }}' + signingEsrpClientId: '${{ parameters.signingEsrpClientId }}' + signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + + dependencies: [] + fileVersion: '$(sqlServerFileVersion)' + packageFullName: 'Microsoft.SqlServer.Server' + packageShortName: 'SqlServer' + versionPropertySuffix: 'SqlServer' + packageVersion: '$(sqlServerPackageVersion)' # ==================================================================== # Stage 2: Abstractions package (depends on Logging from Stage 1) # Abstractions must build after Logging since it has a package # dependency on Internal.Logging. # ==================================================================== - - ${{ if eq(parameters.buildSqlClient, true) }}: - - stage: build_abstractions - displayName: "Build Abstractions Package" - dependsOn: build_independent + - stage: build_abstractions + displayName: "Build Abstractions Package" + dependsOn: + - compute_versions + - build_independent - jobs: - - template: /eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml@self - parameters: - packageName: Abstractions - packageFullName: Microsoft.Data.SqlClient.Extensions.Abstractions - packageVersion: $(effectiveAbstractionsVersion) - versionProperties: >- - -p:AbstractionsPackageVersion=$(effectiveAbstractionsVersion) - -p:AbstractionsAssemblyFileVersion=$(abstractionsAssemblyFileVersion) - -p:LoggingPackageVersion=$(effectiveLoggingVersion) - assemblyFileVersion: $(abstractionsAssemblyFileVersion) - publishSymbols: ${{ parameters.publishSymbols }} - esrpConnectedServiceName: $(ESRPConnectedServiceName) - esrpClientId: $(ESRPClientId) - appRegistrationClientId: $(AppRegistrationClientId) - appRegistrationTenantId: $(AppRegistrationTenantId) - authAkvName: $(AuthAKVName) - authSignCertName: $(AuthSignCertName) - downloadArtifacts: - - artifactName: $(loggingArtifactsName) - displayName: Logging Package + variables: + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] + + jobs: + # Build Microsoft.Data.SqlClient.Extensions.Abstractions + - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self + parameters: + apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Abstractions/dlls' + apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Abstractions/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' + shouldSignPackage: ${{ parameters.isOfficial }} + signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + signingAuthAkvName: '${{ parameters.signingAuthAkvName }}' + signingAuthSignCertName: '${{ parameters.signingAuthSignCertName }}' + signingEsrpClientId: '${{ parameters.signingEsrpClientId }}' + signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + + dependencies: + - artifactName: '${{ parameters.loggingArtifactsName }}' + shortName: 'Logging' + version: '$(sqlClientPackageVersion)' + fileVersion: '$(sqlClientFileVersion)' + packageFullName: 'Microsoft.Data.SqlClient.Extensions.Abstractions' + packageShortName: 'Abstractions' + versionPropertySuffix: 'SqlClient' + packageVersion: '$(sqlClientPackageVersion)' # ==================================================================== # Stage 3: Core packages (depend on Abstractions) - # MDS and Extensions.Azure build in parallel after Abstractions. + # SqlClient and Extensions.Azure build in parallel after Abstractions. # Stage name kept as 'build_dependent' for validate job compatibility. # ==================================================================== - - ${{ if eq(parameters.buildSqlClient, true) }}: - - stage: build_dependent - displayName: "Build Core Packages" - dependsOn: build_abstractions + - stage: build_dependent + displayName: "Build Core Packages" + dependsOn: + - compute_versions + - build_abstractions - jobs: - - template: /eng/pipelines/onebranch/jobs/build-signed-sqlclient-package-job.yml@self - parameters: - publishSymbols: ${{ parameters.publishSymbols }} - isPreview: ${{ parameters.isPreview }} - # TODO: This job should use the effective versions for Abstractions, Logging, - # SqlServer, and SqlClient. + variables: + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] - - template: /eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml@self - parameters: - packageName: Azure - packageFullName: Microsoft.Data.SqlClient.Extensions.Azure - packageVersion: $(effectiveAzureVersion) - versionProperties: >- - -p:AzurePackageVersion=$(effectiveAzureVersion) - -p:AzureAssemblyFileVersion=$(azureAssemblyFileVersion) - -p:AbstractionsPackageVersion=$(effectiveAbstractionsVersion) - -p:LoggingPackageVersion=$(effectiveLoggingVersion) - assemblyFileVersion: $(azureAssemblyFileVersion) - publishSymbols: ${{ parameters.publishSymbols }} - esrpConnectedServiceName: $(ESRPConnectedServiceName) - esrpClientId: $(ESRPClientId) - appRegistrationClientId: $(AppRegistrationClientId) - appRegistrationTenantId: $(AppRegistrationTenantId) - authAkvName: $(AuthAKVName) - authSignCertName: $(AuthSignCertName) - downloadArtifacts: - - artifactName: $(abstractionsArtifactsName) - displayName: Abstractions Package - - artifactName: $(loggingArtifactsName) - displayName: Logging Package + jobs: + # Build Microsoft.Data.SqlClient + - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self + parameters: + apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient/dlls' + apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' + shouldSignPackage: ${{ parameters.isOfficial }} + signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + signingAuthAkvName: '${{ parameters.signingAuthAkvName }}' + signingAuthSignCertName: '${{ parameters.signingAuthSignCertName }}' + signingEsrpClientId: '${{ parameters.signingEsrpClientId }}' + signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + + dependencies: + - artifactName: '${{ parameters.abstractionsArtifactsName }}' + shortName: 'Abstractions' + version: '$(sqlClientPackageVersion)' + - artifactName: '${{ parameters.loggingArtifactsName }}' + shortName: 'Logging' + version: '$(sqlClientPackageVersion)' + # When SqlServer is built this run, depend on the freshly-built artifact; otherwise + # depend on the most recently published SqlServer package (restored from NuGet, so no + # artifact download). + - ${{ if eq(parameters.buildSqlServer, true) }}: + - artifactName: '${{ parameters.sqlServerArtifactsName }}' + shortName: 'SqlServer' + version: '$(sqlServerPackageVersion)' + - ${{ else }}: + - artifactName: '' + shortName: 'SqlServer' + version: '$(sqlServerPackageVersion)' + fileVersion: '$(sqlClientFileVersion)' + packageFullName: 'Microsoft.Data.SqlClient' + packageShortName: 'SqlClient' + versionPropertySuffix: 'SqlClient' + packageVersion: '$(sqlClientPackageVersion)' + + # Build Microsoft.Data.SqlClient.Extensions.Azure + - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self + parameters: + apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Azure/dlls' + apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Azure/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' + shouldSignPackage: ${{ parameters.isOfficial }} + signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + signingAuthAkvName: '${{ parameters.signingAuthAkvName }}' + signingAuthSignCertName: '${{ parameters.signingAuthSignCertName }}' + signingEsrpClientId: '${{ parameters.signingEsrpClientId }}' + signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + + dependencies: + - artifactName: '${{ parameters.abstractionsArtifactsName }}' + shortName: 'Abstractions' + version: '$(sqlClientPackageVersion)' + - artifactName: '${{ parameters.loggingArtifactsName }}' + shortName: 'Logging' + version: '$(sqlClientPackageVersion)' + fileVersion: '$(sqlClientFileVersion)' + packageFullName: 'Microsoft.Data.SqlClient.Extensions.Azure' + packageShortName: 'Azure' + versionPropertySuffix: 'SqlClient' + packageVersion: '$(sqlClientPackageVersion)' # ==================================================================== # Stage 4: Add-on packages (depend on core packages) - # AKV Provider builds after MDS completes. + # Akv Provider builds after SqlClient completes. # ==================================================================== - - ${{ if and(eq(parameters.buildAKVProvider, true), eq(parameters.buildSqlClient, true)) }}: - - stage: build_addons - displayName: "Build Add-on Packages" - dependsOn: build_dependent + - stage: build_addons + displayName: "Build Add-on Packages" + dependsOn: + - compute_versions + - build_dependent - jobs: - - template: /eng/pipelines/onebranch/jobs/build-signed-csproj-package-job.yml@self - parameters: - packageName: AkvProvider - packageFullName: Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider - packageVersion: $(effectiveAkvProviderVersion) - versionProperties: >- - -p:AkvPackageVersion=$(effectiveAkvProviderVersion) - -p:AkvAssemblyFileVersion=$(akvAssemblyFileVersion) - -p:MdsPackageVersion=$(effectiveSqlClientVersion) - -p:LoggingPackageVersion=$(effectiveLoggingVersion) - -p:AbstractionsPackageVersion=$(effectiveAbstractionsVersion) - assemblyFileVersion: $(akvAssemblyFileVersion) - publishSymbols: ${{ parameters.publishSymbols }} - esrpConnectedServiceName: $(ESRPConnectedServiceName) - esrpClientId: $(ESRPClientId) - appRegistrationClientId: $(AppRegistrationClientId) - appRegistrationTenantId: $(AppRegistrationTenantId) - authAkvName: $(AuthAKVName) - authSignCertName: $(AuthSignCertName) - downloadArtifacts: - - artifactName: $(sqlClientArtifactsName) - displayName: SqlClient Package - - artifactName: $(abstractionsArtifactsName) - displayName: Abstractions Package - - artifactName: $(loggingArtifactsName) - displayName: Logging Package + variables: + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] + + jobs: + - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self + parameters: + apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/dlls' + apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' + shouldSignPackage: ${{ parameters.isOfficial }} + signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' + signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' + signingAuthAkvName: '${{ parameters.signingAuthAkvName }}' + signingAuthSignCertName: '${{ parameters.signingAuthSignCertName }}' + signingEsrpClientId: '${{ parameters.signingEsrpClientId }}' + signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' + + dependencies: + - artifactName: '${{ parameters.abstractionsArtifactsName }}' + shortName: 'Abstractions' + version: '$(sqlClientPackageVersion)' + - artifactName: '${{ parameters.loggingArtifactsName }}' + shortName: 'Logging' + version: '$(sqlClientPackageVersion)' + - artifactName: '${{ parameters.sqlClientArtifactsName }}' + shortName: 'SqlClient' + version: '$(sqlClientPackageVersion)' + # When SqlServer is built this run, depend on the freshly-built artifact; otherwise + # depend on the most recently published SqlServer package (restored from NuGet, so no + # artifact download). + - ${{ if eq(parameters.buildSqlServer, true) }}: + - artifactName: '${{ parameters.sqlServerArtifactsName }}' + shortName: 'SqlServer' + version: '$(sqlServerPackageVersion)' + - ${{ else }}: + - artifactName: '' + shortName: 'SqlServer' + version: '$(sqlServerPackageVersion)' + fileVersion: '$(sqlClientFileVersion)' + packageFullName: 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' + packageShortName: 'AkvProvider' + versionPropertySuffix: 'SqlClient' + packageVersion: '$(sqlClientPackageVersion)' # ==================================================================== - # Validation + # Stage 5: Validation + # Validates every package produced by this run, together, so that + # cross-package rules (shared family version, dependency agreement) + # are actually exercised. Depends on all build stages. # ==================================================================== - - ${{ if eq(parameters.buildSqlClient, true) }}: - - stage: mds_package_validation - displayName: "MDS Package Validation" - dependsOn: build_dependent - jobs: - - template: /eng/pipelines/onebranch/jobs/validate-signed-package-job.yml@self - parameters: - artifactName: $(sqlClientArtifactsName) - isPreview: ${{ parameters.isPreview }} + - stage: package_validation + displayName: "Validate Packages" + dependsOn: + - compute_versions + - build_independent + - build_abstractions + - build_dependent + - build_addons + + variables: + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlServerFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerFileVersion'] ] + + jobs: + - template: /eng/pipelines/onebranch/jobs/validate-packages-job.yml@self + parameters: + abstractionsArtifactsName: '${{ parameters.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ parameters.akvProviderArtifactsName }}' + azureArtifactsName: '${{ parameters.azureArtifactsName }}' + loggingArtifactsName: '${{ parameters.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ parameters.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ parameters.sqlServerArtifactsName }}' + + sqlClientPackageVersion: '$(sqlClientPackageVersion)' + sqlClientFileVersion: '$(sqlClientFileVersion)' + sqlServerPackageVersion: '$(sqlServerPackageVersion)' + sqlServerFileVersion: '$(sqlServerFileVersion)' + + buildSqlServer: ${{ parameters.buildSqlServer }} + isOfficial: ${{ parameters.isOfficial }} diff --git a/eng/pipelines/onebranch/stages/compute-versions-stage.yml b/eng/pipelines/onebranch/stages/compute-versions-stage.yml new file mode 100644 index 0000000000..9514c0d56e --- /dev/null +++ b/eng/pipelines/onebranch/stages/compute-versions-stage.yml @@ -0,0 +1,80 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Compute Versions Stage +# ====================== +# Computes package versions up-front in a single fast job. Downstream stages +# consume these via stage/job output variables, e.g.: +# +# stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] +# +# Design: +# - The SqlClient family (Logging, Abstractions, SqlClient, Azure, AKV Provider) shares a single +# version and always uses its "next" version (from Versions.props + BuildNumber). +# - Microsoft.SqlServer.Server is versioned separately: it uses its "next" version when it is +# being built this run, or its "published" (last shipped to NuGet) version when it is not built +# (so the SqlClient family depends on the most recently published SqlServer package). +# - Runs use their "next" version from Versions.props. The pipeline build number is appended to +# prerelease package versions (7.1.0-preview3.26238.3) and its date segment becomes the +# file-version build number (7.1.0.26238), matching the shape shipped by earlier previews. +# - Versions are extracted via build.proj GetVersions* targets by parsing stdout. +# +# This stage MUST run before all build stages so they can consume computed versions. + +parameters: + # Whether Microsoft.SqlServer.Server is being built this run. This drives the SqlServer version + # selection (next when built, published when not); the SqlClient family always uses its next + # version. When SqlServer is not built, the SqlClient family depends on the published SqlServer + # package restored from NuGet. + - name: buildSqlServer + type: boolean + +stages: + - stage: compute_versions + displayName: "Compute Package Versions" + + jobs: + - job: compute_versions_job + displayName: "Extract versions from Versions.props" + pool: + type: linux + + variables: + ob_outputDirectory: $(Build.SourcesDirectory)/no_publish + # Disable all SDL scanning — this job only evaluates MSBuild properties. + ob_sdl_apiscan_enabled: false + ob_sdl_binskim_break: false + ob_sdl_binskim_enabled: false + # No packages are published here, so there is nothing to describe in an SBOM. + ob_sdl_sbom_enabled: false + + steps: + - pwsh: | + New-Item -Path "$(ob_outputDirectory)" -ItemType Directory -Force + "**" | Out-File -FilePath "$(ob_outputDirectory)/.artifactignore" -Encoding ascii + displayName: 'Suppress artifact publishing' + + # Install the global.json-pinned .NET SDK before running any dotnet build, so this stage + # does not rely on whatever SDK happens to be on the agent image. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + # Restore dotnet local tools (pwsh, apicompat, etc.) required by build.proj targets that + # may run during the GetVersions* evaluation. + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + # Extract canonical versions, resolve the effective built/published values, and publish + # the downstream stage output variables. + - task: PowerShell@2 + displayName: "Compute Effective Versions" + name: "versions" + inputs: + targetType: filePath + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/compute-versions.ps1 + arguments: >- + -ProjectPath "$(Build.SourcesDirectory)/build.proj" + -BuildNumber "$(Build.BuildNumber)" + -BuildSqlServer $${{ parameters.buildSqlServer }} diff --git a/eng/pipelines/onebranch/stages/publish-symbols-stage.yml b/eng/pipelines/onebranch/stages/publish-symbols-stage.yml new file mode 100644 index 0000000000..9dd6a5757a --- /dev/null +++ b/eng/pipelines/onebranch/stages/publish-symbols-stage.yml @@ -0,0 +1,187 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Unified symbols publishing stage template for sqlclient OneBranch pipelines. +# Consumed by both the official and non-official pipeline definitions. +# +# This stage publishes PDB symbol files to the internal and public symbol servers. +# Each package's PDBs are published in a separate job to maintain unique naming and +# versioning information per package. +# +# PDBs are expected to be located under the 'symbols/' directory at the root of +# each build artifact, with target framework subdirectories preserved by the build +# job's CopyFiles step. The publish-symbols job consumes that 'symbols' folder directly. +# +# Note that none of the resource DLLs we build produce PDBs, so there are no patterns below that +# match them. +# +# The stage is excluded at compile time when publishSymbols is false. + +parameters: + # ── General parameters ───────────────────────────────────────────────── + + # True to publish symbols (controls whether this stage is emitted at all). + - name: publishSymbols + type: boolean + + # ── Build parameters (used to conditionally include per-package jobs) ─── + + # Whether Microsoft.SqlServer.Server was built this run (its symbols are only published when it + # was built). The SqlClient family is always built, so its symbols are always published. + - name: buildSqlServer + type: boolean + + # ── Package Parameters ───────────────────────────────────────────────── + + - name: abstractionsArtifactsName + type: string + + - name: akvProviderArtifactsName + type: string + + - name: azureArtifactsName + type: string + + - name: loggingArtifactsName + type: string + + - name: sqlClientArtifactsName + type: string + + - name: sqlServerArtifactsName + type: string + + # ── Symbols Publishing Parameters ────────────────────────────────────── + + - name: symbolsAzureSubscription + type: string + + - name: symbolsPublishProjectName + type: string + + - name: symbolsPublishServer + type: string + + - name: symbolsPublishTokenUri + type: string + + - name: symbolsUploadAccount + type: string + + # ── Pre-computed versions (from compute-versions stage) ───────────────── + # These default to $(variableName) tokens that resolve at runtime from + # stage-level variables populated via stageDependencies.compute_versions outputs. + # The SqlClient family shares the SqlClient version; SqlServer is separate. + + - name: sqlClientPackageVersion + type: string + default: $(sqlClientPackageVersion) + + - name: sqlServerPackageVersion + type: string + default: $(sqlServerPackageVersion) + +stages: + # Stage is emitted whenever publishSymbols is true. The SqlClient family is always built, so its + # symbols are always published; SqlServer symbols are published only when it was built. + - ${{ if eq(parameters.publishSymbols, true) }}: + - stage: publish_symbols + displayName: "Publish Symbols" + + # The SqlClient family always builds, so depend on all of its build stages. SqlServer is + # built within build_independent, which we always depend on as well. + dependsOn: + - compute_versions + - build_independent + - build_abstractions + - build_dependent + - build_addons + + variables: + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + + jobs: + # ── Logging ──────────────────────────────────────────────────── + - template: /eng/pipelines/onebranch/jobs/publish-symbols-job.yml@self + parameters: + artifactName: '${{ parameters.loggingArtifactsName }}' + packageFullName: Microsoft.Data.SqlClient.Internal.Logging + packageShortName: Logging + packageVersion: '${{ parameters.sqlClientPackageVersion }}' + symbolsAzureSubscription: '${{ parameters.symbolsAzureSubscription }}' + symbolsPublishProjectName: '${{ parameters.symbolsPublishProjectName }}' + symbolsPublishServer: '${{ parameters.symbolsPublishServer }}' + symbolsPublishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + symbolsUploadAccount: '${{ parameters.symbolsUploadAccount }}' + + # ── SqlServer.Server ─────────────────────────────────────────── + - ${{ if eq(parameters.buildSqlServer, true) }}: + - template: /eng/pipelines/onebranch/jobs/publish-symbols-job.yml@self + parameters: + artifactName: '${{ parameters.sqlServerArtifactsName }}' + packageFullName: Microsoft.SqlServer.Server + packageShortName: SqlServer + packageVersion: '${{ parameters.sqlServerPackageVersion }}' + symbolsAzureSubscription: '${{ parameters.symbolsAzureSubscription }}' + symbolsPublishProjectName: '${{ parameters.symbolsPublishProjectName }}' + symbolsPublishServer: '${{ parameters.symbolsPublishServer }}' + symbolsPublishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + symbolsUploadAccount: '${{ parameters.symbolsUploadAccount }}' + + # ── Abstractions ─────────────────────────────────────────────── + - template: /eng/pipelines/onebranch/jobs/publish-symbols-job.yml@self + parameters: + artifactName: '${{ parameters.abstractionsArtifactsName }}' + packageFullName: Microsoft.Data.SqlClient.Extensions.Abstractions + packageShortName: Abstractions + packageVersion: '${{ parameters.sqlClientPackageVersion }}' + symbolsAzureSubscription: '${{ parameters.symbolsAzureSubscription }}' + symbolsPublishProjectName: '${{ parameters.symbolsPublishProjectName }}' + symbolsPublishServer: '${{ parameters.symbolsPublishServer }}' + symbolsPublishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + symbolsUploadAccount: '${{ parameters.symbolsUploadAccount }}' + + # ── SqlClient ────────────────────────────────────────────────── + - template: /eng/pipelines/onebranch/jobs/publish-symbols-job.yml@self + parameters: + artifactName: '${{ parameters.sqlClientArtifactsName }}' + packageFullName: Microsoft.Data.SqlClient + packageShortName: SqlClient + packageVersion: '${{ parameters.sqlClientPackageVersion }}' + symbolsAzureSubscription: '${{ parameters.symbolsAzureSubscription }}' + symbolsPublishProjectName: '${{ parameters.symbolsPublishProjectName }}' + symbolsPublishServer: '${{ parameters.symbolsPublishServer }}' + symbolsPublishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + symbolsUploadAccount: '${{ parameters.symbolsUploadAccount }}' + + # ── Azure ────────────────────────────────────────────────────── + - template: /eng/pipelines/onebranch/jobs/publish-symbols-job.yml@self + parameters: + artifactName: '${{ parameters.azureArtifactsName }}' + packageFullName: Microsoft.Data.SqlClient.Extensions.Azure + packageShortName: Azure + packageVersion: '${{ parameters.sqlClientPackageVersion }}' + symbolsAzureSubscription: '${{ parameters.symbolsAzureSubscription }}' + symbolsPublishProjectName: '${{ parameters.symbolsPublishProjectName }}' + symbolsPublishServer: '${{ parameters.symbolsPublishServer }}' + symbolsPublishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + symbolsUploadAccount: '${{ parameters.symbolsUploadAccount }}' + + # ── AKV Provider ─────────────────────────────────────────────── + - template: /eng/pipelines/onebranch/jobs/publish-symbols-job.yml@self + parameters: + artifactName: '${{ parameters.akvProviderArtifactsName }}' + packageFullName: Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider + packageShortName: AkvProvider + packageVersion: '${{ parameters.sqlClientPackageVersion }}' + symbolsAzureSubscription: '${{ parameters.symbolsAzureSubscription }}' + symbolsPublishProjectName: '${{ parameters.symbolsPublishProjectName }}' + symbolsPublishServer: '${{ parameters.symbolsPublishServer }}' + symbolsPublishTokenUri: '${{ parameters.symbolsPublishTokenUri }}' + symbolsUploadAccount: '${{ parameters.symbolsUploadAccount }}' diff --git a/eng/pipelines/onebranch/stages/release-stages.yml b/eng/pipelines/onebranch/stages/release-stages.yml index 51eea6e5f0..7ee8023cd9 100644 --- a/eng/pipelines/onebranch/stages/release-stages.yml +++ b/eng/pipelines/onebranch/stages/release-stages.yml @@ -23,15 +23,6 @@ # - false → NuGet Test feed (via NuGetServiceConnectionTest). # # This template depends on stages defined by the build-stages.yml template. -# -# This template depends on the following runtime (i.e. macro expansion) variables being defined: -# -# - effectiveSqlServerVersion -# - effectiveLoggingVersion -# - effectiveAbstractionsVersion -# - effectiveSqlClientVersion -# - effectiveAzureVersion -# - effectiveAkvProviderVersion parameters: # ── General parameters ───────────────────────────────────────────────── @@ -60,24 +51,44 @@ parameters: - production - test - # ── Release parameters ───────────────────────────────────────────────── + # Package Parameters ----------------------------------------------------- - - name: releaseSqlServerServer - type: boolean + - name: abstractionsArtifactsName + type: string - - name: releaseLogging - type: boolean + - name: akvProviderArtifactsName + type: string - - name: releaseAbstractions - type: boolean + - name: azureArtifactsName + type: string - - name: releaseSqlClient + - name: loggingArtifactsName + type: string + + - name: sqlClientArtifactsName + type: string + + - name: sqlServerArtifactsName + type: string + + # ── Symbols publishing parameter ─────────────────────────────────────── + # When true, the release stage will depend on publish_symbols so that + # symbol publishing completes before packages are released. + + - name: publishSymbols type: boolean + default: false - - name: releaseAzure + # ── Release parameters ───────────────────────────────────────────────── + + # Whether to publish the Microsoft.SqlServer.Server package (versioned separately). + - name: releaseSqlServer type: boolean - - name: releaseAKVProvider + # Whether to publish the SqlClient family. The family (Internal.Logging, + # Extensions.Abstractions, Microsoft.Data.SqlClient, Extensions.Azure, and the + # AlwaysEncrypted.AzureKeyVaultProvider) is always released together at the shared version. + - name: releaseSqlClient type: boolean stages: @@ -94,22 +105,25 @@ stages: # true → NuGet Production feed. # false → NuGet Test feed. # ==================================================================== - - ${{ if or(parameters.releaseSqlServerServer, parameters.releaseLogging, parameters.releaseAbstractions, parameters.releaseSqlClient, parameters.releaseAzure, parameters.releaseAKVProvider) }}: + - ${{ if or(parameters.releaseSqlServer, parameters.releaseSqlClient) }}: - stage: release_${{ parameters.stageNameSuffix }} ${{ if eq(parameters.releaseToProduction, true) }}: displayName: Release to NuGet Production ${{ else }}: displayName: Release to NuGet Test dependsOn: - - ${{ if or(parameters.releaseSqlServerServer, parameters.releaseLogging) }}: + # Nothing is published unless every produced package passed validation. This stage also + # depends on all four build stages, but we keep the complete list here anyway to prevent + # regressions if package validation changes its dependencies. + - package_validation + - ${{ if or(parameters.releaseSqlServer, parameters.releaseSqlClient) }}: - build_independent - - ${{ if parameters.releaseAbstractions }}: + - ${{ if parameters.releaseSqlClient }}: - build_abstractions - - ${{ if or(parameters.releaseSqlClient, parameters.releaseAzure) }}: - build_dependent - - mds_package_validation - - ${{ if parameters.releaseAKVProvider }}: - build_addons + - ${{ if parameters.publishSymbols }}: + - publish_symbols variables: - name: onebranchReleaseEnvironment @@ -149,62 +163,52 @@ stages: value: ' (NuGet Test)' jobs: - - ${{ if eq(parameters.releaseSqlServerServer, true) }}: + - ${{ if eq(parameters.releaseSqlServer, true) }}: - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self parameters: packageName: Microsoft.SqlServer.Server - artifactName: drop_build_independent_build_package_SqlServer - packagePath: Microsoft.SqlServer.Server.$(effectiveSqlServerVersion).nupkg + artifactName: '${{ parameters.sqlServerArtifactsName }}' nugetServiceConnection: ${{ variables.nugetServiceConnection }} isProduction: ${{ parameters.isOfficial }} displaySuffix: ${{ variables.nugetTargetSuffix }} - - ${{ if eq(parameters.releaseLogging, true) }}: + - ${{ if eq(parameters.releaseSqlClient, true) }}: - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self parameters: packageName: Microsoft.Data.SqlClient.Internal.Logging - artifactName: drop_build_independent_build_package_Logging - packagePath: Microsoft.Data.SqlClient.Internal.Logging.$(effectiveLoggingVersion).nupkg + artifactName: '${{ parameters.loggingArtifactsName }}' nugetServiceConnection: ${{ variables.nugetServiceConnection }} isProduction: ${{ parameters.isOfficial }} displaySuffix: ${{ variables.nugetTargetSuffix }} - - ${{ if eq(parameters.releaseAbstractions, true) }}: - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self parameters: packageName: Microsoft.Data.SqlClient.Extensions.Abstractions - artifactName: drop_build_abstractions_build_package_Abstractions - packagePath: Microsoft.Data.SqlClient.Extensions.Abstractions.$(effectiveAbstractionsVersion).nupkg + artifactName: '${{ parameters.abstractionsArtifactsName }}' nugetServiceConnection: ${{ variables.nugetServiceConnection }} isProduction: ${{ parameters.isOfficial }} displaySuffix: ${{ variables.nugetTargetSuffix }} - - ${{ if eq(parameters.releaseSqlClient, true) }}: - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self parameters: packageName: Microsoft.Data.SqlClient - artifactName: drop_build_dependent_build_package_SqlClient - packagePath: Microsoft.Data.SqlClient.$(effectiveSqlClientVersion).nupkg + artifactName: '${{ parameters.sqlClientArtifactsName }}' nugetServiceConnection: ${{ variables.nugetServiceConnection }} isProduction: ${{ parameters.isOfficial }} displaySuffix: ${{ variables.nugetTargetSuffix }} - - ${{ if eq(parameters.releaseAzure, true) }}: - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self parameters: packageName: Microsoft.Data.SqlClient.Extensions.Azure - artifactName: drop_build_dependent_build_package_Azure - packagePath: Microsoft.Data.SqlClient.Extensions.Azure.$(effectiveAzureVersion).nupkg + artifactName: '${{ parameters.azureArtifactsName }}' nugetServiceConnection: ${{ variables.nugetServiceConnection }} isProduction: ${{ parameters.isOfficial }} displaySuffix: ${{ variables.nugetTargetSuffix }} - - ${{ if eq(parameters.releaseAKVProvider, true) }}: - template: /eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml@self parameters: packageName: Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider - artifactName: drop_build_addons_build_package_AkvProvider - packagePath: Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider.$(effectiveAkvProviderVersion).nupkg + artifactName: '${{ parameters.akvProviderArtifactsName }}' nugetServiceConnection: ${{ variables.nugetServiceConnection }} isProduction: ${{ parameters.isOfficial }} displaySuffix: ${{ variables.nugetTargetSuffix }} diff --git a/eng/pipelines/onebranch/steps/build-all-configurations-signed-dlls-step.yml b/eng/pipelines/onebranch/steps/build-all-configurations-signed-dlls-step.yml deleted file mode 100644 index b1429e1fd3..0000000000 --- a/eng/pipelines/onebranch/steps/build-all-configurations-signed-dlls-step.yml +++ /dev/null @@ -1,58 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# -parameters: - - # The assembly file version to apply to the Abstractions package. - - name: abstractionsAssemblyFileVersion - type: string - - # The version to apply to the Abstractions package. - - name: abstractionsPackageVersion - type: string - - # The assembly file version to apply to the Logging package. - - name: loggingAssemblyFileVersion - type: string - - # The version to apply to the Logging package. - - name: loggingPackageVersion - type: string - - # The assembly file version to apply to the Mds package. - - name: mdsAssemblyFileVersion - type: string - - # The version to apply to the Mds package. - - name: mdsPackageVersion - type: string - -steps: - # Download our signing key. - - task: DownloadSecureFile@1 - displayName: 'Download Key Pair' - inputs: - secureFile: netfxKeypair.snk - name: keyFile - - # Install the .NET SDK. - - template: /eng/pipelines/steps/install-dotnet.yml@self - - - task: MSBuild@1 - displayName: 'BuildAllConfigurations using build.proj' - inputs: - solution: '**/build.proj' - configuration: Release - msbuildArguments: >- - -t:BuildAllConfigurations - -p:ReferenceType=Package - -p:GenerateNuget=false - -p:SigningKeyPath=$(keyFile.secureFilePath) - -p:AssemblyFileVersion=${{ parameters.mdsAssemblyFileVersion }} - -p:MdsPackageVersion=${{ parameters.mdsPackageVersion }} - -p:AbstractionsPackageVersion=${{ parameters.abstractionsPackageVersion }} - -p:AbstractionsAssemblyFileVersion=${{ parameters.abstractionsAssemblyFileVersion }} - -p:LoggingPackageVersion=${{ parameters.loggingPackageVersion }} - -p:LoggingAssemblyFileVersion=${{ parameters.loggingAssemblyFileVersion }} diff --git a/eng/pipelines/onebranch/steps/build-buildproj-step.yml b/eng/pipelines/onebranch/steps/build-buildproj-step.yml new file mode 100644 index 0000000000..961b972f8c --- /dev/null +++ b/eng/pipelines/onebranch/steps/build-buildproj-step.yml @@ -0,0 +1,83 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This collection of steps to build a project via the build.proj. This will execute the "Build*" +# target in build.proj, where * is the packageShortName provided in the parameters. +# +# Note: This differs from the pr/ci build-buildproj-step.yml in that it always strong-name signs +# the assemblies, it only builds in package reference mode, and as such allows for version +# parameters to be provided. + +parameters: + # Build configuration - Release or Debug. + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # Optional arguments to pass to msbuild to indicate what version of dependencies should be used. + # They should be in the form of "-p:PackageVersionFooBar=1.2.3 ...". + - name: dependencyArguments + type: string + default: '' + + # Project to build. This short name will be appended to "Build" to generate the appropriate + # target to build from build.proj. + - name: packageShortName + type: string + values: + - Azure + - AkvProvider + - Abstractions + - Logging + - SqlClient + - SqlServer + + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion + type: string + + # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this + # package's version. build.proj recognizes only two such properties: PackageVersionSqlClient + # (shared by the entire SqlClient family: Logging, Abstractions, SqlClient, Azure, and the AKV + # Provider) and PackageVersionSqlServer (Microsoft.SqlServer.Server). Provided by the caller. + # Examples: 'SqlClient' -> -p:PackageVersionSqlClient=7.1.0-preview3 + # 'SqlServer' -> -p:PackageVersionSqlServer=1.0.0 + - name: versionPropertySuffix + type: string + + # Version to stamp on the package. Combined with versionPropertySuffix to form the msbuild + # argument, e.g. -p:PackageVersionSqlClient=7.1.0-preview3. + # Always required — compute up-front via the compute-versions stage. + - name: packageVersion + type: string + +steps: + # Download the strong name signing key from secure file storage + - task: DownloadSecureFile@1 + displayName: 'Download Signing Key' + inputs: + secureFile: 'netfxKeypair.snk' + name: keyFile + + - task: DotNetCoreCLI@2 + displayName: 'build.proj - Build${{ parameters.packageShortName }}' + inputs: + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- + -t:Build${{ parameters.packageShortName }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:ReferenceType=Package + -p:SkipDependencyPack=true + -p:SigningKeyPath="$(keyFile.secureFilePath)" + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" + -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" + ${{ parameters.dependencyArguments }} + + - script: tree /a /f $(BUILD_OUTPUT) + displayName: Output Build Output Tree diff --git a/eng/pipelines/onebranch/steps/code-analyze-step.yml b/eng/pipelines/onebranch/steps/code-analyze-step.yml deleted file mode 100644 index c112493603..0000000000 --- a/eng/pipelines/onebranch/steps/code-analyze-step.yml +++ /dev/null @@ -1,46 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# This template defines a step to run Roslyn Analyzers against builds driven by -# build.proj. It is used for both the full Microsoft.Data.SqlClient (MDS) build -# and for individual extension packages; callers control the analyzed targets -# via the msBuildArguments parameter. -# It uses the RoslynAnalyzers@3 task from the Secure Development Team's SDL -# extension: -# -# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-mohanb/security-integration/guardian-wiki/sdl-azdo-extension/roslyn-analyzers-build-task -# -# GOTCHA: This step will clobber any existing build output. It should be run -# _before_ any build steps that perform versioning or signing. - -parameters: - # Source Root. - - name: sourceRoot - type: string - default: $(REPO_ROOT) - - # MSBuild arguments appended after the project file path. The default targets - # BuildAllConfigurations for the full MDS build; override this when analyzing - # individual extension packages. - - name: msBuildArguments - type: string - default: >- - -t:BuildAllConfigurations - -p:configuration=Release - -p:GenerateNuget=false - -p:BuildTools=false - -steps: - - task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@3 - displayName: Roslyn Analyzers - inputs: - msBuildVersion: 17.0 - msBuildArchitecture: x64 - setupCommandLinePicker: vs2022 - msBuildCommandLine: >- - msbuild - ${{parameters.sourceRoot}}\build.proj - ${{parameters.msBuildArguments}} diff --git a/eng/pipelines/onebranch/steps/compound-build-csproj-step.yml b/eng/pipelines/onebranch/steps/compound-build-csproj-step.yml deleted file mode 100644 index 1f1fee00c4..0000000000 --- a/eng/pipelines/onebranch/steps/compound-build-csproj-step.yml +++ /dev/null @@ -1,50 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# Generic build step for csproj-based packages. Each project uses a build.proj target that runs -# Build only and produces assemblies within $(BUILD_OUTPUT). Downstream ESRP DLL signing must -# locate the assemblies within $(BUILD_OUTPUT) for all target frameworks that the csproj targets. -# NuGet packaging is done separately via compound-pack-csproj-step.yml after DLL signing. - -parameters: - # The MSBuild build target in build.proj (e.g. BuildLogging, BuildAbstractions, - # BuildAzure). - - name: buildTarget - type: string - - # Build Configuration. - - name: buildConfiguration - type: string - values: - - Debug - - Release - - # Additional MSBuild arguments for version properties, e.g. - # -p:LoggingPackageVersion=1.0.0 -p:LoggingAssemblyFileVersion=1.0.123 - - name: versionProperties - type: string - default: '' - -steps: - - task: DownloadSecureFile@1 - displayName: Download Signing Key - inputs: - secureFile: netfxKeypair.snk - name: keyFile - - - task: MSBuild@1 - displayName: Build.proj - ${{ parameters.buildTarget }} - inputs: - solution: $(REPO_ROOT)/build.proj - configuration: ${{ parameters.buildConfiguration }} - msbuildArguments: >- - -t:${{ parameters.buildTarget }} - -p:ReferenceType=Package - -p:SigningKeyPath=$(keyFile.secureFilePath) - ${{ parameters.versionProperties }} - - - script: tree /a /f $(BUILD_OUTPUT) - displayName: List Build Output Tree diff --git a/eng/pipelines/onebranch/steps/compound-nuget-pack-step.yml b/eng/pipelines/onebranch/steps/compound-nuget-pack-step.yml deleted file mode 100644 index ef1f3b946a..0000000000 --- a/eng/pipelines/onebranch/steps/compound-nuget-pack-step.yml +++ /dev/null @@ -1,86 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -parameters: - # The C# build configuration to use (e.g. Debug or Release). - - name: buildConfiguration - type: string - values: - - Debug - - Release - - - name: generateSymbolsPackage - type: boolean - - - name: packageVersion - type: string - - - name: nuspecPath - type: string - - - name: outputDirectory - type: string - - # The C# project reference type to use when building and packing the packages. - - name: referenceType - type: string - values: - # Reference sibling packages as NuGet packages. - - Package - # Reference sibling packages as C# projects. - - Project - - # Semi-colon separated properties to pass to nuget via the -properties argument. - - name: properties - type: string - default: '' - -steps: - # This tool is failing on OneBranch pipelines, possibly due to new - # network isolation rules: - # - # ERR:Client network socket disconnected before secure TLS connection was established - # - # Our AKV Official build uses this 1ES image: - # - # Image: 1ES-OB-2022-D8-Netlock-V2_westus2_1_image - # - # An ICM for this issue exists: - # - # https://portal.microsofticm.com/imp/v5/incidents/details/690355343/summary - # - # Recommendation is to remove this step since NuGet is already present on - # the 1ES images. - # - # - task: NuGetToolInstaller@1 - # displayName: 'Install Latest Nuget' - # inputs: - # checkLatest: true - - - ${{ if parameters.generateSymbolsPackage }}: - - task: NuGetCommand@2 - displayName: 'Generate NuGet Package and Symbols Package' - inputs: - command: custom - arguments: >- - pack - ${{ parameters.nuspecPath }} - -Symbols - -SymbolPackageFormat snupkg - -Version ${{ parameters.packageVersion }} - -OutputDirectory ${{ parameters.outputDirectory }} - -Properties "COMMITID=$(Build.SourceVersion);Configuration=${{ parameters.buildConfiguration }};ReferenceType=${{ parameters.referenceType }};${{ parameters.properties }}" - - ${{ else }}: - - task: NuGetCommand@2 - displayName: 'Generate NuGet Package' - inputs: - command: custom - arguments: >- - pack - ${{ parameters.nuspecPath }} - -Version ${{ parameters.packageVersion }} - -OutputDirectory ${{ parameters.outputDirectory }} - -Properties "COMMITID=$(Build.SourceVersion);Configuration=${{ parameters.buildConfiguration }};ReferenceType=${{ parameters.referenceType }};${{ parameters.properties }}" diff --git a/eng/pipelines/onebranch/steps/compound-pack-csproj-step.yml b/eng/pipelines/onebranch/steps/compound-pack-csproj-step.yml deleted file mode 100644 index db9b9a8237..0000000000 --- a/eng/pipelines/onebranch/steps/compound-pack-csproj-step.yml +++ /dev/null @@ -1,42 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# Generic pack step for csproj-based Extension packages (Logging, Abstractions, -# Azure). This step runs the Pack target after DLLs have been signed. The -# NoBuild=true property ensures the DLLs are not rebuilt. - -parameters: - # The MSBuild pack target in build.proj (e.g. PackLogging, PackAbstractions, - # PackAzure). - - name: packTarget - type: string - - # Build Configuration. - - name: buildConfiguration - type: string - values: - - Debug - - Release - - # Additional MSBuild arguments for version properties. - - name: versionProperties - type: string - default: '' - -steps: - - task: MSBuild@1 - displayName: Build.proj - ${{ parameters.packTarget }} - inputs: - solution: $(REPO_ROOT)/build.proj - configuration: ${{ parameters.buildConfiguration }} - msbuildArguments: >- - -t:${{ parameters.packTarget }} - -p:ReferenceType=Package - -p:PackagesDir=$(PACK_OUTPUT)/ - ${{ parameters.versionProperties }} - - - script: tree /a /f $(PACK_OUTPUT) - displayName: List Pack Output Tree After Pack diff --git a/eng/pipelines/onebranch/steps/compound-publish-symbols-step.yml b/eng/pipelines/onebranch/steps/compound-publish-symbols-step.yml deleted file mode 100644 index 1b2e3cb3b9..0000000000 --- a/eng/pipelines/onebranch/steps/compound-publish-symbols-step.yml +++ /dev/null @@ -1,162 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# For more details, see https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL - -parameters: - # Name of the symbols artifact that will be published - - name: artifactName - type: string - - # Azure subscription where the publishing task will execute - - name: azureSubscription - type: string - - # Package name, typically the name of the nuget package being built - - name: packageName - type: string - - # Project that symbols will belong to (decided during symbols onboarding) - - name: publishProjectName - type: string - - # Where symbols publishing service is hosted, will be prepended to trafficmanager.net - - name: publishServer - type: string - - # Whether to publish the uploaded symbols to the internal symbols servers - - name: publishToInternal - type: boolean - - # Whether to publish the uploaded symbols to the public symbols servers - - name: publishToPublic - type: boolean - - # URI to use for requesting a bearer-token for publishing the symbols - - name: publishTokenUri - type: string - - # The C# project reference type to use when building and packing the packages. - - name: referenceType - type: string - values: - # Reference sibling packages as NuGet packages. - - Package - # Reference sibling packages as C# projects. - - Project - - # Pattern to use to search for pdb symbols files to upload/publish - - name: searchPattern - type: string - - # Account/org where the symbols will be uploaded - - name: uploadAccount - type: string - - # Version of the symbols to publish, typically the same as the NuGet package version - - name: version - type: string - -steps: - # Set variable for downstream tasks (allegedly). - # - # Note: Because variables cannot be set in top-level of template, this has to be done during - # runtime. - # - - script: 'echo ##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]${{ parameters.uploadAccount }}' - displayName: 'Set ArtifactServices.Symbol.AccountName to ${{ parameters.uploadAccount }}' - - - task: PublishSymbols@2 - displayName: 'Upload symbols to ${{ parameters.uploadAccount }} org' - inputs: - IndexSources: false - Pat: '$(System.AccessToken)' - SearchPattern: '${{ parameters.searchPattern }}' - SymbolExpirationInDays: 1825 # 5 years - SymbolServerType: 'TeamServices' - SymbolsArtifactName: '${{ parameters.artifactName }}' - SymbolsFolder: '$(BUILD_OUTPUT)/${{ parameters.referenceType }}/bin' - SymbolsMaximumWaitTime: 60 - SymbolsProduct: '${{ parameters.packageName }}' - SymbolsVersion: '${{ parameters.version }}' - - - task: AzureCLI@2 - displayName: 'Publish Symbols' - inputs: - azureSubscription: '${{ parameters.azureSubscription }}' - scriptLocation: inlineScript - scriptType: ps - inlineScript: | - # Propagate parameters to PS variables ################################################ - $artifactName = "${{ parameters.artifactName }}" - echo "artifactName= $artifactName" - - $publishProjectName = "${{ parameters.publishProjectName }}" - echo "publishProjectName= $publishProjectName" - - $publishToInternal = "${{ parameters.publishToInternal }}".ToLower() - echo "publishToInternal= $publishToInternal" - - $publishToPublic = "${{ parameters.publishToPublic }}".ToLower() - echo "publishToPublic= $publishToPublic" - - $publishServer = "${{ parameters.publishServer }}" - echo "publishServer= $publishServer" - - $publishTokenUri = "${{ parameters.publishTokenUri }}" - echo "publishTokenUri= $publishTokenUri" - - # Publish symbols ##################################################################### - # 1) Get the access token for the symbol publishing service - echo "> 1.Acquiring symbol publishing token..." - $symbolPublishingToken = az account get-access-token --resource $publishTokenUri --query accessToken -o tsv - echo "> 1.Symbol publishing token acquired." - - # 2) Register the request name - echo "> 2.Registering request name..." - $requestNameRegistrationBody = "{'requestName': '$artifactName'}" - Invoke-RestMethod ` - -Method POST ` - -Uri "https://$publishServer.trafficmanager.net/projects/$publishProjectName/requests" ` - -Headers @{ Authorization = "Bearer $symbolPublishingToken" } ` - -ContentType "application/json" ` - -Body $requestNameRegistrationBody - echo "> 2.Request name registered successfully." - - # 3) Publish the symbols - echo "> 3.Submitting request to publish symbols..." - $publishSymbolsBody = "{'publishToInternalServer': $publishToInternal, 'publishToPublicServer': $publishToPublic}" - Invoke-RestMethod ` - -Method POST ` - -Uri "https://$publishServer.trafficmanager.net/projects/$publishProjectName/requests/$artifactName" ` - -Headers @{ Authorization = "Bearer $symbolPublishingToken" } ` - -ContentType "application/json" ` - -Body $publishSymbolsBody - echo "> 3.Request to publish symbols submitted successfully." - - # The following REST calls are used to check publishing status. - echo "> 4.Checking the status of the request ..." - Invoke-RestMethod ` - -Method GET ` - -Uri "https://$publishServer.trafficmanager.net/projects/$publishProjectName/requests/$artifactName" ` - -Headers @{ Authorization = "Bearer $symbolPublishingToken" } ` - -ContentType "application/json" - - echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." - - echo "PublishingStatus" - echo "-----------------" - echo "0 NotRequested; The request has not been requested to publish." - echo "1 Submitted; The request is submitted to be published" - echo "2 Processing; The request is still being processed" - echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" - - echo "PublishingResult" - echo "-----------------" - echo "0 Pending; The request has not completed or has not been requested." - echo "1 Succeeded; The request has published successfully" - echo "2 Failed; The request has failed to publish" - echo "3 Cancelled; The request was cancelled" diff --git a/eng/pipelines/onebranch/steps/esrp-code-signing-step.yml b/eng/pipelines/onebranch/steps/esrp-code-signing-step.yml deleted file mode 100644 index 59322d67aa..0000000000 --- a/eng/pipelines/onebranch/steps/esrp-code-signing-step.yml +++ /dev/null @@ -1,163 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# -parameters: - - name: artifactType - values: - - dll - - pkg - - - name: sourceRoot - type: string - default: $(REPO_ROOT) - - - name: dllPattern - type: string - default: 'Microsoft.Data.SqlClient*.dll' - - - name: artifactDirectory - type: string - default: $(PACK_OUTPUT) - - - name: ESRPConnectedServiceName - type: string - default: $(ESRPConnectedServiceName) - - - name: appRegistrationClientId - type: string - default: $(appRegistrationClientId) - - - name: appRegistrationTenantId - type: string - default: $(appRegistrationTenantId) - - - name: AuthAKVName - type: string - default: $(AuthAKVName) - - - name: AuthSignCertName - type: string - default: $(AuthSignCertName) - - - name: EsrpClientId - type: string - default: $(EsrpClientId) - -steps: -- ${{ if eq(parameters.artifactType, 'dll') }}: - # See: https://aka.ms/esrp.scantask - - task: EsrpMalwareScanning@6 - displayName: 'ESRP MalwareScanning' - inputs: - ConnectedServiceName: '${{parameters.ESRPConnectedServiceName }}' - AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' - AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' - EsrpClientId: '${{parameters.EsrpClientId }}' - UseMSIAuthentication: true - FolderPath: '${{parameters.sourceRoot }}' - Pattern: '${{ parameters.dllPattern }}' - CleanupTempStorage: 1 - VerboseLogin: 1 - - # See: https://aka.ms/esrp.signtask - - task: EsrpCodeSigning@6 - displayName: 'ESRP CodeSigning' - inputs: - ConnectedServiceName: '${{parameters.ESRPConnectedServiceName }}' - AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' - AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' - EsrpClientId: '${{parameters.EsrpClientId }}' - UseMSIAuthentication: true - AuthAKVName: '${{parameters.AuthAKVName }}' - AuthSignCertName: '${{parameters.AuthSignCertName }}' - FolderPath: '${{parameters.sourceRoot }}' - Pattern: '${{ parameters.dllPattern }}' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft Data SqlClient Data Provider for SQL Server" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-230012", - "operationSetCode": "SigntoolVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - -- ${{ if eq(parameters.artifactType, 'pkg') }}: - # See: https://aka.ms/esrp.scantask - - task: EsrpMalwareScanning@6 - displayName: 'ESRP MalwareScanning Nuget Package' - inputs: - ConnectedServiceName: '${{parameters.ESRPConnectedServiceName }}' - AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' - AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' - EsrpClientId: '${{parameters.EsrpClientId }}' - UseMSIAuthentication: true - FolderPath: '${{parameters.artifactDirectory }}' - Pattern: '*.*nupkg' - CleanupTempStorage: 1 - VerboseLogin: 1 - - # See: https://aka.ms/esrp.signtask - - task: EsrpCodeSigning@6 - displayName: 'ESRP CodeSigning Nuget Package' - inputs: - inputs: - ConnectedServiceName: '${{parameters.ESRPConnectedServiceName }}' - AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' - AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' - EsrpClientId: '${{parameters.EsrpClientId }}' - UseMSIAuthentication: true - AuthAKVName: '${{parameters.AuthAKVName }}' - AuthSignCertName: '${{parameters.AuthSignCertName }}' - FolderPath: '${{parameters.artifactDirectory }}' - Pattern: '*.*nupkg' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetSign", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-401405", - "operationSetCode": "NuGetVerify", - "parameters": [ ], - "toolName": "sign", - "toolVersion": "1.0" - } - ] diff --git a/eng/pipelines/onebranch/steps/compound-esrp-dll-signing-step.yml b/eng/pipelines/onebranch/steps/esrp-dll-signing-step.yml similarity index 72% rename from eng/pipelines/onebranch/steps/compound-esrp-dll-signing-step.yml rename to eng/pipelines/onebranch/steps/esrp-dll-signing-step.yml index 44649f94aa..db95ef4f24 100644 --- a/eng/pipelines/onebranch/steps/compound-esrp-dll-signing-step.yml +++ b/eng/pipelines/onebranch/steps/esrp-dll-signing-step.yml @@ -30,8 +30,9 @@ parameters: - name: esrpClientId type: string - # Globbing pattern for the files to sign. All files in $(BUILD_OUTPUT)/Package/bin - # that match this pattern will be scanned and signed. This should end with ".dll". + # Minimatch pattern(s) for the files to sign. All files in $(BUILD_OUTPUT) + # that match will be scanned and signed. Supports multi-line patterns and + # negation (!pattern). See https://aka.ms/esrp.signtask for details. - name: pattern type: string @@ -40,13 +41,14 @@ steps: - task: EsrpMalwareScanning@6 displayName: ESRP DLL Malware Scanning inputs: - AppRegistrationClientId: ${{ parameters.appRegistrationClientId }} - AppRegistrationTenantId: ${{ parameters.appRegistrationTenantId }} + AppRegistrationClientId: '${{ parameters.appRegistrationClientId }}' + AppRegistrationTenantId: '${{ parameters.appRegistrationTenantId }}' CleanupTempStorage: 1 - ConnectedServiceName: ${{ parameters.esrpConnectedServiceName }} - EsrpClientId: ${{ parameters.esrpClientId }} - FolderPath: $(BUILD_OUTPUT)/Package/bin - Pattern: ${{ parameters.pattern }} + ConnectedServiceName: '${{ parameters.esrpConnectedServiceName }}' + EsrpClientId: '${{ parameters.esrpClientId }}' + FolderPath: '$(BUILD_OUTPUT)' + Pattern: '${{ parameters.pattern }}' + UseMinimatch: true UseMSIAuthentication: true VerboseLogin: 1 @@ -54,14 +56,15 @@ steps: - task: EsrpCodeSigning@6 displayName: ESRP DLL Signing inputs: - AppRegistrationClientId: ${{ parameters.appRegistrationClientId }} - AppRegistrationTenantId: ${{ parameters.appRegistrationTenantId }} - AuthAKVName: ${{ parameters.authAkvName }} - AuthSignCertName: ${{ parameters.authSignCertName }} - ConnectedServiceName: ${{ parameters.esrpConnectedServiceName }} - EsrpClientId: ${{ parameters.esrpClientId }} - FolderPath: $(BUILD_OUTPUT)/Package/bin - Pattern: ${{ parameters.pattern }} + AppRegistrationClientId: '${{ parameters.appRegistrationClientId }}' + AppRegistrationTenantId: '${{ parameters.appRegistrationTenantId }}' + AuthAKVName: '${{ parameters.authAkvName }}' + AuthSignCertName: '${{ parameters.authSignCertName }}' + ConnectedServiceName: '${{ parameters.esrpConnectedServiceName }}' + EsrpClientId: '${{ parameters.esrpClientId }}' + FolderPath: '$(BUILD_OUTPUT)' + Pattern: '${{ parameters.pattern }}' + UseMinimatch: true signConfigType: inlineSignParams UseMSIAuthentication: true inlineOperation: | diff --git a/eng/pipelines/onebranch/steps/compound-esrp-nuget-signing-step.yml b/eng/pipelines/onebranch/steps/esrp-nuget-signing-step.yml similarity index 82% rename from eng/pipelines/onebranch/steps/compound-esrp-nuget-signing-step.yml rename to eng/pipelines/onebranch/steps/esrp-nuget-signing-step.yml index 34e903465f..6a6682d40f 100644 --- a/eng/pipelines/onebranch/steps/compound-esrp-nuget-signing-step.yml +++ b/eng/pipelines/onebranch/steps/esrp-nuget-signing-step.yml @@ -30,6 +30,16 @@ parameters: - name: esrpClientId type: string + # Folder path to search for NuGet packages to sign + - name: searchPath + type: string + + # Minimatch pattern(s) to search for NuGet packages. Supports multi-line + # patterns and negation (!pattern). Defaults to '*.*nupkg'. + - name: searchPattern + type: string + default: '*.*nupkg' + steps: # See: https://aka.ms/esrp.scantask - task: EsrpMalwareScanning@6 @@ -40,8 +50,9 @@ steps: CleanupTempStorage: 1 ConnectedServiceName: '${{ parameters.esrpConnectedServiceName }}' EsrpClientId: '${{ parameters.esrpClientId }}' - FolderPath: '$(PACK_OUTPUT)' - Pattern: '*.*nupkg' + FolderPath: '${{ parameters.searchPath }}' + Pattern: '${{ parameters.searchPattern }}' + UseMinimatch: true UseMSIAuthentication: true VerboseLogin: 1 @@ -55,8 +66,9 @@ steps: EsrpClientId: '${{ parameters.esrpClientId }}' AuthAKVName: '${{ parameters.authAkvName }}' AuthSignCertName: '${{ parameters.authSignCertName }}' - FolderPath: '$(PACK_OUTPUT)' - Pattern: '*.*nupkg' + FolderPath: '${{ parameters.searchPath }}' + Pattern: '${{ parameters.searchPattern }}' + UseMinimatch: true signConfigType: 'inlineSignParams' UseMSIAuthentication: true inlineOperation: | diff --git a/eng/pipelines/onebranch/steps/pack-buildproj-step.yml b/eng/pipelines/onebranch/steps/pack-buildproj-step.yml new file mode 100644 index 0000000000..b8ee6e72dd --- /dev/null +++ b/eng/pipelines/onebranch/steps/pack-buildproj-step.yml @@ -0,0 +1,87 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Generic pack step for build.proj packages. This step runs the Pack* target to generate a NuGet +# package of the target package. +# +# Note: This step assumes that the package has been built previously using a Build* target. +# `-p:PackBuild=false` will be passed to msbuild to ensure the previously built assemblies are +# not trampled. + +parameters: + # Build configuration - Release or Debug. + - name: buildConfiguration + type: string + values: + - Debug + - Release + + # Optional arguments to pass to msbuild to indicate what version of dependencies should be used. + # They should be in the form of "-p:PackageVersionFooBar=1.2.3 ...". + - name: dependencyArguments + type: string + default: '' + + # The full name of the package. This will be used to generate paths when copying NuGet packages + # out of the build output. + - name: packageFullName + type: string + values: + - Microsoft.Data.SqlClient + - Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider + - Microsoft.Data.SqlClient.Extensions.Abstractions + - Microsoft.Data.SqlClient.Extensions.Azure + - Microsoft.Data.SqlClient.Internal.Logging + - Microsoft.SqlServer.Server + + # Project to build. This short name will be appended to "Pack" to generate the appropriate + # target to package via build.proj. + - name: packageShortName + type: string + values: + - Abstractions + - AkvProvider + - Azure + - Logging + - SqlClient + - SqlServer + + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion + type: string + + # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this + # package's version. build.proj recognizes only two such properties: PackageVersionSqlClient + # (shared by the entire SqlClient family: Logging, Abstractions, SqlClient, Azure, and the AKV + # Provider) and PackageVersionSqlServer (Microsoft.SqlServer.Server). Provided by the caller. + # Examples: 'SqlClient' -> -p:PackageVersionSqlClient=7.1.0-preview3 + # 'SqlServer' -> -p:PackageVersionSqlServer=1.0.0 + - name: versionPropertySuffix + type: string + + # Version to stamp on the package. Combined with versionPropertySuffix to form the msbuild + # argument, e.g. -p:PackageVersionSqlClient=7.1.0-preview3. + # Always required — compute up-front via the compute-versions stage. + - name: packageVersion + type: string + +steps: + - task: DotNetCoreCLI@2 + displayName: 'build.proj - Pack${{ parameters.packageShortName }}' + inputs: + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- + -t:Pack${{ parameters.packageShortName }} + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackBuild=false + -p:ReferenceType=Package + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" + -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" + ${{ parameters.dependencyArguments }} + + - script: tree /a /f $(BUILD_OUTPUT) + displayName: Output Build Output Tree diff --git a/eng/pipelines/onebranch/steps/publish-symbols-step.yml b/eng/pipelines/onebranch/steps/publish-symbols-step.yml index a23dbf4556..dc794f480d 100644 --- a/eng/pipelines/onebranch/steps/publish-symbols-step.yml +++ b/eng/pipelines/onebranch/steps/publish-symbols-step.yml @@ -1,119 +1,154 @@ -#################################################################################### -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -# # -# doc: https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL # -#################################################################################### +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Symbols Publishing Pipeline +# =========================== +# Canonical docs: https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL +# +# Publishing symbols to SymWeb (internal) and MSDL (public) is a two-step process: +# +# Step 1 - Upload: The PublishSymbols@2 task uploads PDB files to the Azure DevOps +# symbol store under an artifact name (SymbolsArtifactName). This stores the +# symbols but does NOT make them available on SymWeb or MSDL. +# +# Step 2 - Publish: The publish-symbols.ps1 script calls the Symbols Publishing Pipeline +# REST API to request that the previously uploaded symbols be published to the +# internal (SymWeb) and/or public (MSDL) symbol servers. +# +# IMPORTANT: Step 2 depends on Step 1. The ArtifactName parameter passed to the publish +# script MUST match the SymbolsArtifactName used by PublishSymbols@2 so that both steps +# reference the same uploaded artifact. + parameters: + # Name of the symbols artifact that will be published + - name: artifactName + type: string + + # Azure subscription where the publishing task will execute + - name: azureSubscription + type: string + + # Package name, typically the name of the nuget package being built + - name: packageName + type: string + + # Project that symbols will belong to (decided during symbols onboarding) + - name: publishProjectName + type: string + + # Where symbols publishing service is hosted, will be prepended to trafficmanager.net + - name: publishServer + type: string + + # Whether to publish the uploaded symbols to the internal symbols servers + - name: publishToInternal + type: boolean + + # Whether to publish the uploaded symbols to the public symbols servers + - name: publishToPublic + type: boolean + + # URI to use for requesting a bearer-token for publishing the symbols + - name: publishTokenUri + type: string + + # Pattern to use to search for pdb symbols files to upload/publish + - name: searchPattern + type: string + + # Root folder to search for PDB files. When called from a build job this is typically + # $(BUILD_OUTPUT); when called from the dedicated symbols stage it points at the + # downloaded artifact path containing the PDBs. + - name: symbolsFolder + type: string + default: '$(BUILD_OUTPUT)' + + # Account/org where the symbols will be uploaded + - name: uploadAccount + type: string - # The full name of the package whose symbols are being published. - - name: packageFullName - type: string - - # The version of the package whose symbols are being published. - - name: packageVersion - type: string - - # Our symbols account name. - - name: symbolsAccount - type: string - default: SqlClientDrivers - - # The symbols server to publish to. - - name: symbolServer - type: string - default: $(SymbolServer) - - # The token URI for the symbol publishing service. - - name: symbolTokenUri - type: string - default: $(SymbolTokenUri) - - # A pair of flags indicating whether to publish to the internal and public symbol servers. Both - # default to true. - - name: publishToServers - type: object - default: - internal: true - public: true + # Version of the symbols to publish, typically the same as the NuGet package version + - name: version + type: string steps: -- pwsh: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]${{parameters.symbolsAccount}}"' - displayName: 'Set ArtifactServices.Symbol.AccountName to ${{parameters.symbolsAccount}}' - -- pwsh: 'Write-Host "##vso[task.setvariable variable=symbolsArtifactName;]${{ parameters.packageFullName }}_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_${{ parameters.packageVersion }}_$(System.TimelineId)"' - displayName: 'Set symbolsArtifactName variable' - -- task: PublishSymbols@2 - displayName: 'Upload symbols to ${{parameters.symbolsAccount }} org' - inputs: - SymbolsFolder: '$(Build.SourcesDirectory)\artifacts\Package\bin' - SearchPattern: '**/*.pdb' - IndexSources: false - SymbolServerType: TeamServices - SymbolsMaximumWaitTime: 60 - SymbolExpirationInDays: 1825 # 5 years - SymbolsProduct: ${{ parameters.packageFullName }} - SymbolsVersion: ${{ parameters.packageVersion }} - SymbolsArtifactName: $(symbolsArtifactName) - Pat: $(System.AccessToken) - -- task: AzureCLI@2 - displayName: 'Publish symbols' - inputs: - azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net' - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - $publishToInternalServer = "${{parameters.publishToServers.internal }}".ToLower() - $publishToPublicServer = "${{parameters.publishToServers.public }}".ToLower() - - echo "Publishing request name: $(symbolsArtifactName)" - echo "Publish to internal server: $publishToInternalServer" - echo "Publish to public server: $publishToPublicServer" - - $symbolServer = "${{parameters.symbolServer }}" - $tokenUri = "${{parameters.symbolTokenUri }}" - # Registered project name in the symbol publishing pipeline: https://portal.microsofticm.com/imp/v3/incidents/incident/520844254/summary - $projectName = "Microsoft.Data.SqlClient.SNI" - - # Get the access token for the symbol publishing service - $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv - - echo "> 1.Symbol publishing token acquired." - - echo "Registering the request name ..." - $requestName = "$(symbolsArtifactName)" - $requestNameRegistrationBody = "{'requestName': '$requestName'}" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody - - echo "> 2.Registration of request name succeeded." - - echo "Publishing the symbols ..." - $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" - echo "Publishing symbols request body: $publishSymbolsBody" - Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody - - echo "> 3.Request to publish symbols succeeded." - - # The following REST calls are used to check publishing status. - echo "> 4.Checking the status of the request ..." - - Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" - - echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." - - echo "PublishingStatus" - echo "-----------------" - echo "0 NotRequested; The request has not been requested to publish." - echo "1 Submitted; The request is submitted to be published" - echo "2 Processing; The request is still being processed" - echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" - - echo "PublishingResult" - echo "-----------------" - echo "0 Pending; The request has not completed or has not been requested." - echo "1 Succeeded; The request has published successfully" - echo "2 Failed; The request has failed to publish" - echo "3 Cancelled; The request was cancelled" + # NOTE: ArtifactServices.Symbol.AccountName is set as a job-level variable in + # publish-symbols-job.yml. On OneBranch Linux agents, PublishSymbols@2 runs on the host (outside + # the build container) due to 1ES PT credential isolation. A ##vso[task.setvariable] inside the + # container is not visible to host-level tasks, so the variable must be declared at job scope. + # + # Reference: + # https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL#Option_B:_OneBranch + + # Log the PDB files that match the search pattern so we can verify no unexpected files are + # included in the upload. + - pwsh: | + $folder = '${{ parameters.symbolsFolder }}' + $glob = '${{ parameters.searchPattern }}' + Write-Host "Symbols folder : $folder" + Write-Host "Search pattern : $glob" + + # Convert the glob to a regex that can match against relative paths. + # ** → match any number of path segments (.*?) + # * → match within a single segment ([^/\\]*) + # ? → match a single non-separator char ([^/\\]) + # . → literal dot + $regex = [regex]::Escape($glob) + $regex = $regex -replace '\\\*\\\*[/\\]?', '.*?' # ** or **/ + $regex = $regex -replace '\\\*', '[^/\\]*' # single * + $regex = $regex -replace '\\\?', '[^/\\]' # single ? + $regex = '^' + $regex + '$' + Write-Host "Regex : $regex" + Write-Host "" + + $allFiles = Get-ChildItem -Path $folder -Recurse -File + $matched = $allFiles | Where-Object { + $rel = $_.FullName.Substring($folder.Length).TrimStart('/\') + $rel -match $regex + } + + if (-not $matched) { + Write-Host "##vso[task.logissue type=warning]No PDB files matched pattern '$glob' under '$folder'" + } else { + $count = @($matched).Count + Write-Host "Matched $count PDB file(s):" + $matched | ForEach-Object { Write-Host " $($_.FullName)" } + } + displayName: 'Log PDBs matching ${{ parameters.searchPattern }}' + + # Step 1 - Upload: Push PDB files to the Azure DevOps symbol store. + # The SymbolsArtifactName set here is the key that links this upload to Step 2. + - task: PublishSymbols@2 + displayName: 'Step 1: Upload symbols to ${{ parameters.uploadAccount }} org' + inputs: + IndexSources: false + Pat: '$(System.AccessToken)' + SearchPattern: '${{ parameters.searchPattern }}' + SymbolExpirationInDays: 1825 # 5 years + SymbolServerType: 'TeamServices' + SymbolsArtifactName: '${{ parameters.artifactName }}' + SymbolsFolder: '${{ parameters.symbolsFolder }}' + SymbolsMaximumWaitTime: 60 + SymbolsProduct: '${{ parameters.packageName }}' + SymbolsVersion: '${{ parameters.version }}' + + # Step 2 - Publish: Request the Symbols Publishing Pipeline to publish the uploaded + # symbols to SymWeb and/or MSDL. The -ArtifactName argument must match the + # SymbolsArtifactName from Step 1. + - task: AzureCLI@2 + displayName: 'Step 2: Publish symbols to SymWeb/MSDL' + inputs: + azureSubscription: '${{ parameters.azureSubscription }}' + scriptType: pscore + scriptLocation: scriptPath + scriptPath: '$(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/publish-symbols.ps1' + arguments: >- + -PublishServer "${{ parameters.publishServer }}" + -PublishTokenUri "${{ parameters.publishTokenUri }}" + -PublishProjectName "${{ parameters.publishProjectName }}" + -ArtifactName "${{ parameters.artifactName }}" + -PublishToInternal $${{ parameters.publishToInternal }} + -PublishToPublic $${{ parameters.publishToPublic }} diff --git a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml new file mode 100644 index 0000000000..6e9b0e8106 --- /dev/null +++ b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml @@ -0,0 +1,195 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This template runs Roslyn Analyzers (SDL) against a build.proj target using the RoslynAnalyzers@3 +# task from the Secure Development Team's SDL extension, in "Copy Logs Only" mode: +# +# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-mohanb/security-integration/guardian-wiki/sdl-azdo-extension/roslyn-analyzers-build-task +# +# PROVENANCE: Every statement in this file about how the RoslynAnalyzers task behaves is current as +# of task version v3 (RoslynAnalyzers@3, 3.289.0) and was verified against concrete evidence -- the +# actual pipeline run logs, the task definition (task.json / inputMap.json), the gdn-task-lib task +# source, and the Guardian RoslynAnalyzers CLI binaries (Microsoft.Guardian.RoslynAnalyzers*.dll). +# Re-verify these claims if the task's major version changes. +# +# HOW IT WORKS (integrated analyzers + Copy Logs Only): +# .NET [Roslyn] security analyzers are compiler-integrated: they only run as part of the actual +# csc/vbc compilation. build.proj is an orchestrator -- each package's real compile happens in a +# separate "dotnet build .csproj" that build.proj launches with an task. That is +# the crux: anything the RoslynAnalyzers task appends to an *outer* "dotnet build build.proj" +# command (its auto/manual "re-run the build" modes) is an MSBuild global property, and global +# properties do NOT cross an into a child "dotnet build" process. So an injected analyzer +# would only ever see build.proj (which compiles nothing) and never the real projects, producing +# zero results. That is exactly why earlier auto/manual-mode attempts collected 0 SARIF logs. +# +# Instead we use the task's documented "Copy Logs Only" alternative -- integrate the analyzers into +# the build itself, then have the task only collect the results: +# 1. This step runs its own isolated "dotnet build build.proj -t:Build" with +# EnableAnalyzers=true. build.proj forwards that flag into every leaf "dotnet build" it execs +# (via EnableAnalyzersArgument), and src/Directory.Build.props -- which every product project +# imports -- reacts by enabling the full analyzer set and setting ErrorLog to a per-project +# "*.csproj..sarif" log. src/Directory.Build.targets verifies that each leaf compile +# produced its configured log. Because the analyzers and verification are enabled on the leaf +# projects themselves, they run inside the real compiles regardless of the boundary. +# 2. The RoslynAnalyzers@3 task then runs in Copy Logs Only mode (copyLogsOnly: true) and simply +# collects and sanitizes those *.csproj.*.sarif and *.vbproj.*.sarif logs from +# logRootDirectory for SDL/Guardian compliance. It performs no build and no compiler re-run, +# so none of the msBuildVersion / +# msBuildArchitecture / Visual-Studio-setup concerns apply -- the task never needs MSBuild, +# so it is inherently agnostic to the container's VS/MSBuild version (e.g. MSBuild 18 on the +# ltsc2025/vse2026 image). +# +# EnableAnalyzers and IsolatedBuildPath are independent build.proj properties. We set both here: +# EnableAnalyzers turns analysis on; IsolatedBuildPath keeps this analysis build from disturbing +# real build output (see ISOLATION). +# +# WHAT THE TASK ITSELF INJECTS (v3), AND HOW THIS TEMPLATE COVERS IT: +# In its build-driving modes the task appends five MSBuild properties to the compile command and +# injects the analyzers via user-profile ImportBefore/ImportAfter files. This template enables the +# analyzers on the leaf projects instead, reproducing the effects that matter. Item by item: +# +# | Task injection (v3) | Purpose | How this template covers it | +# |----------------------------------------|-------------------------------------|---------------------------------------------------| +# | /p:Features= | Turns on Roslyn IOperation + | latest-recommended: SDK 18 Roslyn has IOperation | +# | "IOperation,flow-analysis" | dataflow so the taint/crypto | on by default, so these rules run. Add | +# | | security rules (CA3xxx/CA5xxx) run. | flow-analysis to | +# | | | Directory.Build.props if any go missing. | +# | /p:CodeAnalysisRuleSet= | Selects the exact SDL rule IDs + | AnalysisLevel=latest-recommended. This is the | +# | ...Sdl.Recommended.Warning.ruleset | severities; disables non-SDL rules. | SDK's own "recommended" mode, NOT the private SDL | +# | | | ruleset, so complete overlap is not guaranteed; | +# | | | see the CAVEAT in Directory.Build.props. The | +# | | | internal IA* rules need the | +# | | | Microsoft.Internal.Analyzers package, which is | +# | | | Microsoft-internal-only and MUST NOT be added to | +# | | | the public governed feed, so they are not | +# | | | reproduced here. See NOTE ON THE INTERNAL IA* | +# | | | RULES below. | +# | /p:TreatWarningsAsErrors=false | Record every diagnostic instead of | false in the | +# | | failing at the first one. | EnableAnalyzers block of Directory.Build.props. | +# | | | Warning-clean compilation is still enforced by | +# | | | the ordinary build later in each job, which does | +# | | | run with TreatWarningsAsErrors=true. | +# | /p:RunCodeAnalysis=false | Disables legacy *binary* FxCop | Already false by default in SDK-style projects; | +# | | (not the Roslyn analyzers). | we never enable it. | +# | /p:GdnRoslynAnalyzersRunId= | Gates the injected props/targets so | Not needed. We enable analyzers directly on the | +# | | they apply only to this build. | leaf projects, so there is no global injection to | +# | | | gate (see the note below). | +# | ImportBefore *.props / ImportAfter | Adds the analyzer assemblies and | Analyzers: EnableNETAnalyzers + | +# | *.targets under %LOCALAPPDATA%\...\ | sets ErrorLog=.sarif. | latest-recommended (no EnforceCodeStyleInBuild). | +# | MSBuild\Current | | ErrorLog: we set | +# | | | $(MSBuildProjectFullPath)....sarif | +# | | | (SARIF v1 -- NO version=2; see the sanitizer | +# | | | note in Directory.Build.props). | +# +# WHY THE TASK'S OWN INJECTION YIELDS 0 SARIF THROUGH build.proj: the ImportAfter *.targets live in +# the user profile, so they ARE imported by build.proj's inner "dotnet build " execs -- but +# they self-gate on $(GdnRoslynAnalyzersRunId), and that property (like CodeAnalysisRuleSet and +# Features) is passed only on the OUTER "dotnet build build.proj" command and does not cross the +# into the child compiles. So the injected targets no-op in the real compiles. Enabling the +# analyzers on the leaf projects (EnableAnalyzers) removes that gate entirely. +# +# NOTE ON THE INTERNAL IA* RULES: +# The SDL-recommended ruleset also contains internal IA* ("Internal Analyzers") rules that ship in +# the Microsoft.Internal.Analyzers package. That package is Microsoft-internal and confidential: it +# is NOT on nuget.org, and it MUST NOT be added to this repo's governed feed +# (sqlclientdrivers.pkgs.visualstudio.com/public/...), which is PUBLIC-scoped -- doing so would +# leak internal tooling and breach its internal-use license. So the leaf-project analysis above +# (AnalysisLevel=latest-recommended) covers the CA* rules but NOT the IA* rules. +# +# The IA* rules can only be run from the PRIVATE ADO.Net project pipelines, where a Microsoft- +# internal NuGet feed is reachable. NuGet.analysis.config adds that feed using an environment- +# variable placeholder and maps Microsoft.Internal.* exclusively to it. Only these analysis builds +# select that config, allowing Microsoft.Internal.Analyzers to be restored and used without +# changing the normal NuGet.config. +# +# ISOLATION: +# This template is self-contained and safe to run at any point in a job -- before or after a real +# build -- because the analysis build writes its binaries to a separate location and never touches +# the real build output, using the IsolatedBuildPath build.proj property. + +parameters: + # Optional arguments to pass to msbuild to indicate what version of dependencies should be used. + # They should be of the form "-p:PackageVersionFooBar=1.2.3 ...". + - name: dependencyArguments + type: string + default: '' + + # Project to build. This short name will be appended to "Build" to generate the appropriate + # target to build from build.proj. + - name: packageShortName + type: string + values: + - Abstractions + - AkvProvider + - Azure + - Logging + - SqlClient + - SqlServer + + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion + type: string + + # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this + # package's version. See build-buildproj-step.yml for the full explanation. + - name: versionPropertySuffix + type: string + + # Version to stamp on the package. Combined with versionPropertySuffix to form the msbuild + # argument, e.g. -p:PackageVersionSqlClient=7.1.0-preview3. + - name: packageVersion + type: string + +steps: + # Step 1: Authenticate to the internal Azure Artifacts feed so the leaf restores can pull + # Microsoft.Internal.Analyzers. NuGetAuthenticate sets up the Azure Artifacts credential provider + # for feeds the build identity can access in this organization. + - task: NuGetAuthenticate@1 + displayName: 'Internal analyzers: authenticate internal feed' + + # Step 2: Isolated analysis build. Compiles the package with EnableAnalyzers=true so that every leaf + # "dotnet build" that build.proj execs turns on the full Roslyn analyzer set and verifies its SARIF. + - task: DotNetCoreCLI@2 + displayName: 'Build for Roslyn analysis - build.proj Build${{ parameters.packageShortName }}' + inputs: + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- + -t:Build${{ parameters.packageShortName }} + -p:Configuration=Release + -p:ReferenceType=Package + -p:SkipDependencyPack=true + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" + -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" + -p:IsolatedBuildPath="$(Agent.TempDirectory)/roslyn" + -p:EnableAnalyzers=true + -p:InternalAnalyzers=true + -p:InternalAnalyzersNugetConfig="$(REPO_ROOT)/NuGet.analysis.config" + -p:InternalAnalyzersVersion=$(InternalAnalyzersVersion) + ${{ parameters.dependencyArguments }} + env: + # dotnet restore expands this environment variable into the NuGet feed URL for + # Microsoft.Internal.Analyzers. + INTERNAL_ANALYZERS_FEED: $(InternalAnalyzersFeed) + + # Step 3: List every SARIF file that the collector will ingest. + - pwsh: | + $sarifFiles = @(Get-ChildItem -Path '$(REPO_ROOT)' -Recurse -File -Include '*.csproj.*.sarif', '*.vbproj.*.sarif' | Sort-Object FullName) + Write-Host "Roslyn collector will ingest $($sarifFiles.Count) SARIF file(s):" + $sarifFiles | ForEach-Object { Write-Host " $($_.FullName)" } + displayName: 'List Roslyn SARIF files for collection' + + # Step 4: Collect the analysis results. In Copy Logs Only mode the task does not build or re-run + # the compiler -- it just gathers and sanitizes the *.csproj.*.sarif and *.vbproj.*.sarif logs + # produced by Step 2 and hands them to Guardian/SDL. + - task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@3 + displayName: 'Roslyn Analyzers (collect) - build.proj Build${{ parameters.packageShortName }}' + inputs: + copyLogsOnly: true + # Root to search for the *.csproj.*.sarif and *.vbproj.*.sarif logs. The analysis build wrote + # them next to each project under the repo checkout; the collector globs this directory + # recursively. + logRootDirectory: '$(REPO_ROOT)' diff --git a/eng/pipelines/onebranch/steps/validate-localization-step.yml b/eng/pipelines/onebranch/steps/validate-localization-step.yml new file mode 100644 index 0000000000..8cb590f6d0 --- /dev/null +++ b/eng/pipelines/onebranch/steps/validate-localization-step.yml @@ -0,0 +1,16 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +steps: + - task: PowerShell@2 + displayName: 'Validate localized resources' + inputs: + targetType: filePath + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/validate-localization.ps1 + arguments: >- + -ResourcesDirectory "$(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/src/Resources" + -AllowlistPath "$(Build.SourcesDirectory)/.config/LocalizationValidationAllowlist.json" diff --git a/eng/pipelines/onebranch/steps/validate-packages-step.yml b/eng/pipelines/onebranch/steps/validate-packages-step.yml new file mode 100644 index 0000000000..173df4d7ed --- /dev/null +++ b/eng/pipelines/onebranch/steps/validate-packages-step.yml @@ -0,0 +1,80 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Builds and runs tools/PackageValidator over a directory of produced NuGet packages. +# +# The validator is invoked once for the whole directory rather than once per package, because its +# most valuable checks are cross-package: it confirms that every package in the SqlClient family +# carries the same version and that their inter-package dependency ranges agree. Running it per +# package would silently skip all of those findings. +# +# Two invocations are made over the same inputs. The first writes a machine-readable report and is +# ungated, so the artifact exists even for a failing run. It still fails the step if the validator +# itself errors, because the report it produced could not then be trusted. The second renders the +# human-readable report and applies the gate, so a failed build shows the findings in its own log. + +parameters: + # Directory scanned recursively for .nupkg files. Sibling .snupkg files must sit beside their + # .nupkg for symbol matching to resolve, which is how the build jobs publish them. + - name: packagesPath + type: string + + # Path of the JSON report to write. + - name: reportPath + type: string + + # Expected versions shared by the whole SqlClient family (Logging, Abstractions, SqlClient, + # Azure, AkvProvider), applied as wildcard expectations. Pointing every package at the same + # value is what proves they agree, and also catches every package being consistently wrong. + - name: sqlClientPackageVersion + type: string + + - name: sqlClientFileVersion + type: string + + # Expected versions for the separately-versioned Microsoft.SqlServer.Server, applied as a per-id + # override of the family wildcard. Left empty when SqlServer is not built this run: its package + # is then absent from the drop, and the validator rejects an expectation with an empty value. + - name: sqlServerPackageVersion + type: string + default: '' + + - name: sqlServerFileVersion + type: string + default: '' + + # Finding severities and/or categories that fail the build. Run the validator with --help to + # see the full set of categories. + - name: failOn + type: object + default: + - error + +steps: + - task: DotNetCoreCLI@2 + displayName: 'build.proj - BuildPackageValidator' + inputs: + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- + -t:BuildPackageValidator + -p:Configuration=Release + + - task: PowerShell@2 + displayName: 'Validate NuGet packages' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/validate-packages.ps1 + arguments: >- + -ValidatorPath "$(REPO_ROOT)/tools/PackageValidator/src/bin/Release/net10.0/PackageValidator.dll" + -PackagesPath "${{ parameters.packagesPath }}" + -ReportPath "${{ parameters.reportPath }}" + -SqlClientPackageVersion "${{ parameters.sqlClientPackageVersion }}" + -SqlClientFileVersion "${{ parameters.sqlClientFileVersion }}" + -SqlServerPackageVersion "${{ parameters.sqlServerPackageVersion }}" + -SqlServerFileVersion "${{ parameters.sqlServerFileVersion }}" + -FailOn "${{ join(',', parameters.failOn) }}" diff --git a/eng/pipelines/onebranch/variables/common-variables.yml b/eng/pipelines/onebranch/variables/common-variables.yml deleted file mode 100644 index feddd2c6ce..0000000000 --- a/eng/pipelines/onebranch/variables/common-variables.yml +++ /dev/null @@ -1,182 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# Common variables shared across all OneBranch Official pipelines. - -variables: - - group: Release Variables - - # Symbols publishing credentials - - group: Symbols publishing - # SymbolsAzureSubscription - # SymbolsPublishProjectName - # SymbolsPublishServer - # SymbolsPublishTokenUri - # SymbolsUploadAccount - - - group: ESRP Federated Creds (AME) - # ESRPConnectedServiceName - # ESRPClientId - # AppRegistrationClientId - # AppRegistrationTenantId - # AuthAKVName - # AuthSignCertName - - - name: CommitHead - value: '' # the value will be extracted from the repo's head - - # Aliases required by compound step templates (compound-esrp-dll-signing-step, - # compound-esrp-nuget-signing-step, compound-nuget-pack-step, etc.). - - # The root of our repo. - - name: REPO_ROOT - value: $(Build.SourcesDirectory) - - # This is where our C# projects place their build outputs (see Directory.Build.props - # ). We exclude the reference type here since Official pipelines always use - # Package mode. - - name: BUILD_OUTPUT - value: $(REPO_ROOT)/artifacts - - # This is where our C# projects place their NuGet package outputs. This is intentionally - # separate from packages/ (where downloaded pipeline artifacts go) so that ESRP signing and - # OneBranch artifact publishing only operate on packages built by the current job. - - name: PACK_OUTPUT - value: $(REPO_ROOT)/output - - # C# assembly versions must be in the format: Major.Minor.Build.Revision, but - # $(Build.BuildNumber) has the format XXX.YY. Additionally, each version part - # must be a positive 16-bit integer less than 65535. Simply concatenating - # both parts of $(Build.BuildNumber) could produce values larger than 65534, - # so we must omit the second part entirely. Unfortunately, this may result - # in multiple subsequent pipline builds using the same C# assembly versions. - # The package versions will not be affected and will show the complete - # $(Build.BuildNumber) values. - - name: assemblyBuildNumber - value: $[ split(variables['Build.BuildNumber'], '.')[0] ] - - # ---------------------------------------------------------------------------- - # Abstractions Package Versions - # - # These are version values that will be used by the official build. They - # should be updated after each release to reflect the next release's versions. - - # The NuGet package version for GA releases (non-preview). - - name: abstractionsPackageVersion - value: '1.0.0' - - # The NuGet package version for preview releases. - - name: abstractionsPackagePreviewVersion - value: 1.0.0-preview1.$(Build.BuildNumber) - - # The AssemblyFileVersion for all assemblies in the Abstractions package. - - name: abstractionsAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # ---------------------------------------------------------------------------- - # MDS Package Versions - # - # These are version values that will be used by the official build. They - # should be updated after each release to reflect the next release's versions. - - # The NuGet package version for GA releases (non-preview). - - name: mdsPackageVersion - value: '7.0.0' - - # The NuGet package version for preview releases. - - name: mdsPackagePreviewVersion - value: 7.0.0-preview4.$(Build.BuildNumber) - - # The AssemblyFileVersion for all assemblies in the MDS package. - - name: mdsAssemblyFileVersion - value: 7.0.0.$(assemblyBuildNumber) - - # The path to the NuGet packaging specification file used to generate the MDS NuGet package. - - name: nuspecPath - value: '$(REPO_ROOT)/tools/specs/Microsoft.Data.SqlClient.nuspec' - - # ---------------------------------------------------------------------------- - # Logging Package Versions - # - # These are version values that will be used by the official build. They - # should be updated after each release to reflect the next release's versions. - - # The NuGet package version for GA releases (non-preview). - - name: loggingPackageVersion - value: '1.0.0' - - # The NuGet package version for preview releases. - - name: loggingPackagePreviewVersion - value: 1.0.0-preview1.$(Build.BuildNumber) - - # The AssemblyFileVersion for all assemblies in the Logging package. - - name: loggingAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # ---------------------------------------------------------------------------- - # Azure Package Versions - # - # These are version values that will be used by the official build. They - # should be updated after each release to reflect the next release's versions. - - # The NuGet package version for GA releases (non-preview). - - name: azurePackageVersion - value: '1.0.0' - - # The NuGet package version for preview releases. - - name: azurePackagePreviewVersion - value: 1.0.0-preview1.$(Build.BuildNumber) - - # The AssemblyFileVersion for all assemblies in the Azure package. - - name: azureAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # ---------------------------------------------------------------------------- - # SqlServer.Server Package Versions - # - # These are version values that will be used by the official build. They - # should be updated after each release to reflect the next release's versions. - - # The NuGet package version for GA releases (non-preview). - - name: sqlServerPackageVersion - value: '1.0.0' - - # The NuGet package version for preview releases. - - name: sqlServerPackagePreviewVersion - value: 1.0.0-preview1.$(Build.BuildNumber) - - # The AssemblyFileVersion for all assemblies in the SqlServer.Server package. - - name: sqlServerAssemblyFileVersion - value: 1.0.0.$(assemblyBuildNumber) - - # ---------------------------------------------------------------------------- - # AKV Provider Package Versions - # - # These are version values that will be used by the official build. They - # should be updated after each release to reflect the next release's versions. - - # The NuGet package version for GA releases (non-preview). - - name: akvPackageVersion - value: '7.0.0' - - # The NuGet package version for preview releases. - - name: akvPackagePreviewVersion - value: 7.0.0-preview1.$(Build.BuildNumber) - - # The AssemblyFileVersion for all assemblies in the AKV Provider package. - - name: akvAssemblyFileVersion - value: 7.0.0.$(assemblyBuildNumber) - - # ---------------------------------------------------------------------------- - # MDS Symbols Publishing - # - # Alias variables for MDS publish-symbols-step.yml, which defaults to - # $(SymbolServer) and $(SymbolTokenUri). These map to the values provided - # by the "Symbols publishing" library group. - - name: SymbolServer - value: $(SymbolsPublishServer) - - name: SymbolTokenUri - value: $(SymbolsPublishTokenUri) diff --git a/eng/pipelines/onebranch/variables/onebranch-variables.yml b/eng/pipelines/onebranch/variables/onebranch-variables.yml index 9c60ff4cf0..b6d4ce7d5c 100644 --- a/eng/pipelines/onebranch/variables/onebranch-variables.yml +++ b/eng/pipelines/onebranch/variables/onebranch-variables.yml @@ -4,52 +4,73 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This file is only included in MDS OneBranch Official pipelines. +# This file is only included in SqlClient OneBranch official/non-official pipelines. variables: - - template: /eng/pipelines/onebranch/variables/common-variables.yml@self + # Libraries ============================================================== - # onebranch template variables - - name: ob_sdl_binskim_break - value: true # https://aka.ms/obpipelines/sdl - - name: Packaging.EnableSBOMSigning - value: true - - name: WindowsContainerImage - value: "onebranch.azurecr.io/windows/ltsc2022/vse2022:latest" # Docker image which is used to build the project https://aka.ms/obpipelines/containers - - # OneBranch automatically publishes pipeline artifacts from all jobs. Use of the - # PublishPipelineArtifacts task is prohibited. All content in the ob_outputDirectory is - # published. The artifacts are named according to this pattern: + # These variables are used for running ESRP signing tasks. All names start with "Signing" to make + # it clear that these variables are used for signing (as opposed to other msc tasks). # - # drop__ + # SigningAppRegistrationClientId + # SigningAppRegistrationTenantId + # SigningAuthAkvName + # SigningAuthSignCertName + # SigningEsrpClientId + # SigningEsrpConnectedServiceName + - group: 'esrp-variables-v2' + + # These variables are used for running symbol publishing tasks. All names start with "Symbols" to + # make it clear that these variables are used for symbols (as opposed to other msc tasks). # - # Downstream stages and jobs may download these artifacts via the DownloadPipelineArtifact task, - # but they must know the artifact name. + # SymbolsAzureSubscription + # SymbolsPublishProjectNameSqlClient + # SymbolsPublishServerProd + # SymbolsPublishServerPpe + # SymbolsPublishTokenUriProd + # SymbolsPublishTokenUriPpe + # SymbolsUploadAccount + - group: 'symbols-variables-v3' + + # These variables point the SDL Roslyn analysis step at the internal Microsoft.Internal.Analyzers + # package (the "IA*" rules). The package is Microsoft-internal and confidential, so the feed URL + # and pinned version live in this ADO.Net-project variable group rather than in the repo. Consumed + # by eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml. # - # Here we define the artifact names that our OneBranch pipelines produce to avoid hardcoding them - # throughout the pipeline files. If any of the stage or job names change, we only need to update - # the artifact names here. + # InternalAnalyzersFeed + # InternalAnalyzersVersion + - group: 'internal-analyzers-variables-v1' + + # Well-Known Variables ################################################### - # The Abstractions package artifacts. - - name: abstractionsArtifactsName - value: drop_build_abstractions_build_package_Abstractions + # Directory where downloaded pipeline artifacts (NuGet packages from earlier + # stages) are placed. Build jobs use this as a local NuGet package source so + # that downstream packages can resolve dependencies on packages built by + # upstream stages. + - name: JOB_INPUT + value: $(REPO_ROOT)/packages - # The AKV Provider package artifacts. - - name: akvArtifactsName - value: drop_build_addons_build_package_AkvProvider + # Root directory for all job output artifacts (NuGet packages, symbols, etc.). OneBranch auto + # publishes everything under this directory as the artifact for the job. + - name: JOB_OUTPUT + value: $(REPO_ROOT)/output - # The Azure package artifacts. - - name: azureArtifactsName - value: drop_build_dependent_build_package_Azure + # OneBranch Template Variables ########################################### - # The Logging package artifacts. - - name: loggingArtifactsName - value: drop_build_independent_build_package_Logging + - name: CommitHead + value: '' # the value will be extracted from the repo's head - # The SqlServer package artifacts. - - name: sqlServerArtifactsName - value: drop_build_independent_build_package_SqlServer + - name: Packaging.EnableSBOMSigning + value: true + + # OneBranch supplies a variety of container images we must use for our jobs. + # + # https://eng.ms/docs/products/onebranch/infrastructureandimages/containerimages/containerimages + + # Windows jobs use this image. + - name: WindowsContainerImage + value: onebranch.azurecr.io/windows/ltsc2025/vse2026:latest - # The SqlClient package artifacts. - - name: sqlClientArtifactsName - value: drop_build_dependent_build_package_SqlClient + # Linux jobs use this image. + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 diff --git a/eng/pipelines/onebranch/variables/package-variables.yml b/eng/pipelines/onebranch/variables/package-variables.yml new file mode 100644 index 0000000000..65fd0867d0 --- /dev/null +++ b/eng/pipelines/onebranch/variables/package-variables.yml @@ -0,0 +1,39 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# This file contains variables that relate to the various packages that are produced via the +# OneBranch official/non-official pipelines. They are grouped by the packages they represent. +# +# ARTIFACT NAMING +# =============== +# OneBranch automatically publishes pipeline artifacts from all jobs named as: +# +# drop__ +# +# Downstream stages download these via DownloadPipelineArtifact and must know the name. + +variables: + # ------------------------------------------------------------------------ + # Artifact Names + # These identify the published pipeline artifacts for inter-stage consumption. + + - name: abstractionsArtifactsName + value: 'drop_build_abstractions_build_package_Abstractions' + + - name: akvProviderArtifactsName + value: 'drop_build_addons_build_package_AkvProvider' + + - name: azureArtifactsName + value: 'drop_build_dependent_build_package_Azure' + + - name: loggingArtifactsName + value: 'drop_build_independent_build_package_Logging' + + - name: sqlClientArtifactsName + value: 'drop_build_dependent_build_package_SqlClient' + + - name: sqlServerArtifactsName + value: 'drop_build_independent_build_package_SqlServer' diff --git a/eng/pipelines/onebranch/variables/sqlclient-validation-variables.yml b/eng/pipelines/onebranch/variables/sqlclient-validation-variables.yml deleted file mode 100644 index 75ad369dbc..0000000000 --- a/eng/pipelines/onebranch/variables/sqlclient-validation-variables.yml +++ /dev/null @@ -1,38 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# This file is only included in MDS OneBranch Official pipelines. - -variables: - - group: Release Variables - - template: /eng/pipelines/onebranch/variables/common-variables.yml@self - - - name: TempFolderName # extract the nuget package here - value: temp - - name: extractedNugetRootPath - value: $(Build.SourcesDirectory)\$(TempFolderName)\Microsoft.Data.SqlClient - - name: extractedNugetPath - value: $(extractedNugetRootPath).$(mdsPackageVersion) - - name: expectedFolderNames - value: lib,ref,runtimes - - name: expectedDotnetVersions - value: netstandard2.0,net462,net8.0,net9.0 - - name: Database - value: Northwind - - name: platform - value: AnyCPU - - name: TargetNetFxVersion - value: net481 - - name: TargetNetCoreVersion - value: net9.0 - - name: SQLTarget - value: localhost - - name: encrypt - value: false - - name: SQL_NP_CONN_STRING - value: Data Source=np:$(SQLTarget);Initial Catalog=$(Database);Integrated Security=true;Encrypt=$(ENCRYPT);TrustServerCertificate=true; - - name: SQL_TCP_CONN_STRING - value: Data Source=tcp:$(SQLTarget);Initial Catalog=$(Database);Integrated Security=true;Encrypt=$(ENCRYPT);TrustServerCertificate=true; diff --git a/eng/pipelines/perf/README.md b/eng/pipelines/perf/README.md new file mode 100644 index 0000000000..467f02b0f5 --- /dev/null +++ b/eng/pipelines/perf/README.md @@ -0,0 +1,473 @@ +# SqlClient Performance Test Pipeline + +This directory contains the Azure DevOps pipeline and supporting scripts that run the +Microsoft.Data.SqlClient [BenchmarkDotNet](https://benchmarkdotnet.org/) performance tests on a +dedicated performance test lab (Azure Dedicated Hosts), compare the branch under test against a +released NuGet baseline, and optionally ingest the results into an Azure Data Explorer (Kusto) +database. + +## Contents + +| Path | Purpose | +| ---- | ------- | +| `sqlclient-perf-pipeline.yml` | The main (manual/nightly) pipeline. Extends `v1/Perf.Test.Job.yml@PerfTemplates`. Baseline = released NuGet package; ingests into Kusto. | +| `sqlclient-perf-pr-pipeline.yml` | PR pipeline. Same template, same scripts, same options; baseline = **`release/7.1` branch source**; **no Kusto ingestion**. | +| `sqlclient-perf-experiment.yml` | Experiment pipeline. Same template, same scripts; both passes build the **same source** and differ only in one runner-config switch; **no Kusto ingestion**. | +| `scripts/run-perf-tests.sh` | Linux on-VM entry point: install SDK, create DB, run benchmarks (interleaved or sequential), compare. Baseline is a released package (`--baseline-version`), another git ref's source (`--baseline-source-ref`), or the same source with one runner-config switch flipped off (`--switch-under-test`). | +| `scripts/run-perf-tests.ps1` | Windows equivalent (ProcessorAffinity instead of `taskset`). | +| `scripts/interleave_perf.py` | Interleaved + best-of-N orchestrator: runs each unit baseline↔candidate back-to-back and confirms regressions across N passes. | +| `scripts/compare_perf.py` | Compares baseline vs current BenchmarkDotNet JSON → delta (md + json). Reused by the orchestrator. | +| `scripts/perf_to_kusto.py` | Translates BenchmarkDotNet "full" JSON → Kusto `PerfRun` + `PerfBenchmarkResult` NDJSON. | +| `scripts/ingest_kusto.py` | Queued Kusto ingestion (az CLI auth, runs on the agent). | + +## Architecture + +``` +Queue pipeline (ADO) + │ + ▼ +extends: v1/Perf.Test.Job.yml@PerfTemplates + │ (provisions a dedicated-host VM, SCPs the repo, runs the script over SSH, + │ SCPs back, publishes it, tears the VM down) + ▼ +ON THE VM ── run-perf-tests.{sh,ps1} + 1. Install the .NET SDK pinned by global.json (+ runtimes). + 2. Create the perf database on the VM's SQL Server. + 3. Inject the VM SQL connection string into runnerconfig. + 4. Baseline pass → MDS from NuGet.org (Package mode) → results/baseline/ + (PR pipeline: MDS built from the source tree) + 5. Current pass → MDS built from source (ProjectReference) → results/current/ + (both pinned to PERF_CLIENT_CPUS; interleaved per-unit by default, or two full + sequential passes when benchmarkRunMode=sequential) + 6. interleave_perf.py / compare_perf.py → results/comparison/ + results/summary.md + ▼ +ON THE AGENT ── pipeline post-test steps + • Show the BenchmarkDotNet markdown reports in the log. + • perf_to_kusto.py → NDJSON for both passes (published as 'perf-kusto-payloads'). + • (optional) AzureCLI@2 + ingest_kusto.py → Kusto database. + (both Kusto steps are omitted entirely in the PR pipeline) +``` + +The extends template only exposes **post-test** steps to consumers (no pre-build hook), so **both +benchmark passes run inside the on-VM script**. Translation and ingestion run on the **agent** +because that is where the pipeline's AAD identity / service connection and the native pipeline +context variables are available (the VM is behind NAT and lacks the pipeline identity). + +## Parameters + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `platform` | `linux` | `linux` or `windows` VM + client. | +| `dotnetFramework` | `net9.0` | TFM the benchmarks run against (`net8.0`/`net9.0`/`net10.0`). | +| `testTimeoutMinutes` | `180` | Template timeout waiting for the VM run. | +| `baselineVersion` | `7.0.2` | **Baseline Version** — released MDS the branch is compared against. Empty = current-only (no baseline pass / comparison). | +| `regressionThreshold` | `10` | Percent slowdown (current vs baseline mean) flagged as a regression. | +| `failOnRegression` | `false` | When `true`, a candidate-slower regression **fails** the run (gate). In interleaved mode only **confirmed** regressions (best-of-N majority) fail. Default off. | +| `benchmarkRunMode` | `interleaved` | `interleaved` (per-unit baseline↔candidate + best-of-N confirmation) or `sequential` (legacy two full passes). | +| `confirmationRuns` | `3` | Best-of-N: interleaved passes for a flagged unit before a regression is confirmed. `1` disables confirmation. Interleaved mode only. | +| `useManagedSniOnWindows` | `false` | SqlClient `UseManagedSniOnWindows` flag. Applied to the runner config for **both** passes on the VM and recorded in `PerfRun.Config`. Default matches the checked-in `runnerconfig.jsonc`. | +| `useOptimizedAsyncBehaviour` | `true` | SqlClient `UseOptimizedAsyncBehaviour` flag. Applied to both passes and recorded in `PerfRun.Config`. Default matches `runnerconfig.jsonc`. | +| `useConnectionPoolV2` | `false` | SqlClient `UseConnectionPoolV2` flag. Applied to both passes and recorded in `PerfRun.Config`. Default matches `runnerconfig.jsonc`. | +| `enableKustoIngestion` | `true` | **Ingest results into Kusto** — when `false`, the run still benchmarks + compares but skips ingesting into the perf database. When `true`, ingestion additionally requires the `ADX Cluster Variables` group to be populated. | + +The following values are **fixed constants** in the pipeline (not parameters or variables), since they +are invariant for this pipeline: `buildConfiguration = Release`, `sourcesSubDir = dotnet-sqlclient` +(the multi-repo checkout folder for `self`, which must match the ADO repository name), and +`driverName = Microsoft.Data.SqlClient` (recorded on every row as `DriverName` / `DerivedRunId`). + +### Kusto (Azure Data Explorer) ingestion variables + +The ADX ingestion coordinates are **not** pipeline parameters — they come from a pipeline library +variable group named **`ADX Cluster Variables`** so no infrastructure identifiers are hard-coded in +the pipeline. The group must define: + +| Variable | Description | +| -------- | ----------- | +| `KustoClusterUri` | ADX cluster URI, e.g. `https://..kusto.windows.net`. Empty ⇒ ingestion skipped. | +| `KustoDatabase` | Target Kusto database. Empty ⇒ ingestion skipped. | +| `KustoServiceConnection` | Azure DevOps ARM service connection whose SP has ingest rights. Empty ⇒ ingestion skipped. | + +Ingestion is gated at runtime: it only runs when `KustoClusterUri`, `KustoDatabase` and +`KustoServiceConnection` are all non-empty, so the pipeline still runs + compares before the +cluster/service connection exist. + +### Managing the baseline version + +`baselineVersion` is **manually managed**. After each stable release is published to NuGet.org, +bump the `default` in `sqlclient-perf-pipeline.yml` (e.g. `7.0.2` → the next stable). It can also be +overridden at queue time without editing the pipeline. + +## PR pipeline (`sqlclient-perf-pr-pipeline.yml`) + +`sqlclient-perf-pr-pipeline.yml` answers the question a PR author actually has — *does my branch +regress the branch I am merging into?* — by running the **same** benchmarks, on the **same** extends +template, through the **same** on-VM scripts, with the **same** configuration options as the main +pipeline. It differs in exactly two ways: + +| | `sqlclient-perf-pipeline.yml` | `sqlclient-perf-pr-pipeline.yml` | +| --- | --- | --- | +| Candidate | branch the run is queued on | branch the run is queued on (unchanged) | +| Baseline | released NuGet package (`baselineVersion`, default `7.0.2`) | **source of another git ref** (`baselineSourceRef`, default `release/7.1`) | +| Kusto | translates + (optionally) ingests | **never** — no ADX variable group, no translate/ingest steps | + +Both pipelines are **manual / queue-time only** (`pr: none`, `trigger: none`): a run occupies a +dedicated host for hours, so a PR opts into it explicitly. + +PR-only parameters (everything else is identical to the table above): + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `baselineSourceRef` | `release/7.1` | Git ref of this repo whose **source** is the baseline. Empty = current-only (no baseline pass / comparison). | +| `baselineRepoUrl` | `https://github.com/dotnet/SqlClient.git` | Fallback remote used to obtain the baseline ref when the VM copy of the checkout cannot fetch it from its own `origin`. | +| `testTimeoutMinutes` | `210` | Higher than the main pipeline's `180` because **both** sides are built from source. | + +### Source baseline (`--baseline-source-ref` / `-BaselineSourceRef`) + +The run scripts accept the source baseline as an alternative to `--baseline-version` (the two are +mutually exclusive and the script fails fast if both are supplied). On the VM the script: + +1. Materialises the baseline ref next to the checkout, in `../sqlclient-perf-baseline-src` — outside + the checkout so it can never be picked up by the candidate build or the results copy. It first + tries the copied checkout's own `origin` (`git fetch --depth 1` into a private + `refs/remotes/perfbaseline/*` namespace, then `git worktree add --detach`), and falls back to a + shallow `git clone` of `baselineRepoUrl` when the tree arrived without `.git` or `origin` needs + credentials. On ADO the origin fetch is *expected* to fail — the checkout is copied to the VM + without credentials — so the clone fallback is the normal path there. Every network git call runs + non-interactively (`GIT_TERMINAL_PROMPT=0`, no credential helper, stdin closed) under a + `GIT_NET_TIMEOUT_SECS` timeout (default 300s), and its output is kept in + `/diagnostics/git-*.log`, so a missing credential fails in under a second instead of + blocking the job on a prompt. +2. Builds **that ref's own `PerformanceTests` project** as the `baseline` variant, so the driver + under measurement is the baseline ref's source via `ProjectReference` — mirroring how the + `current` variant is built from this branch. No NuGet baseline config is involved. +3. Labels the comparison `"@"` (e.g. `main@a1b2c3d`) so a run states exactly which + baseline commit it was measured against, and writes that label to + `/baseline-label.txt`. The PR pipeline reads that file after the results are copied + back and tags the build `Baseline @`, so the exact baseline commit is visible in + the ADO build list. (The pipeline cannot produce this tag on its own — at compile time it only + knows the ref name, which would tag every run identically.) + +Because each side builds its own harness, a benchmark added by the PR simply shows up as `new` in +the comparison (and one removed by the PR as `removed`) instead of failing the run. Both passes +share the single generated runner config (`RUNNER_CONFIG` / `DATATYPES_CONFIG` env vars), so +connection string and behaviour flags are identical on both sides. + +## Experiment pipeline (`sqlclient-perf-experiment.yml`) + +The three perf pipelines are the same benchmarks, template and scripts pointed at three different +questions. Two of them vary the **source** under measurement; the third varies the **config**: + +| Question | Pipeline | Baseline | Current | +| --- | --- | --- | --- | +| Has this branch regressed against a released package? | `sqlclient-perf-pipeline.yml` | released NuGet package | queued branch | +| Does my PR regress the branch it merges into? | `sqlclient-perf-pr-pipeline.yml` | `release/7.1` source | queued branch | +| What does this switch cost or buy? | `sqlclient-perf-experiment.yml` | queued branch, switch **off** | queued branch, switch **on** | + +`sqlclient-perf-experiment.yml` picks one runner-config switch via the `switchUnderTest` +queue-time parameter (`UseConnectionPoolV2`, `UseOptimizedAsyncBehaviour` or +`UseManagedSniOnWindows`) and runs the baseline pass with it `false` and the current pass with it +`true`. Both passes measure the **same commit** — the branch the run is queued on — so queue it on a +PR branch to ask "what does this switch do to my change?", or on `main` to ask "what does it do to +`main`?". + +Two passes are required because these are `AppContext` switches latched process-wide (for example +`UseConnectionPoolV2` is read and cached the first time a connection pool is created), so they cannot +be toggled between benchmarks within a single process. The pipeline wires that into the existing +comparison machinery — interleaved best-of-N or sequential, via `benchmarkRunMode` — instead of two +ad hoc manual runs. + +Switch-pipeline parameters that differ from the tables above: + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `switchUnderTest` | `UseConnectionPoolV2` | The single switch to A/B. Baseline forces it `false`, current forces it `true`. | +| `failIfSwitchSlower` | `false` | Maps onto the scripts' `--fail-on-regression` gate, but means something different here: "switch on is slower" is usually the *result* you queued the run to measure, not a defect. Enable it only when asserting the switch must not be a slowdown (e.g. before flipping its default). | +| `testTimeoutMinutes` | `180` | Same as the main pipeline, not the PR pipeline's `210`: both sides are the same source, so only **one** driver build is needed. | + +The pipeline deliberately does **not** expose `baselineVersion` / `baselineSourceRef` (the run +scripts reject combining those with `--switch-under-test`, since a simultaneous source change would +make the delta unattributable), nor the `useManagedSniOnWindows` / `useOptimizedAsyncBehaviour` / +`useConnectionPoolV2` flags. Every switch except the one under test stays at its checked-in +`runnerconfig.jsonc` value, so the measured difference is attributable to exactly one variable. + +### Why these runs are never ingested into Kusto + +This mode has its own pipeline file, rather than being a flag on the other two, specifically so that +"never ingested" is structural rather than a conditional someone can flip. Ingesting a switch +experiment would corrupt the perf database three ways: + +* **`DerivedRunId` collision.** The ID is `driver|commit|pipelineRunId`. The other two pipelines keep + their two rows distinct because the baseline row carries a *different* commit (`v7.0.2`, or the + baseline ref's sha). Here both passes are the same commit in the same pipeline run, so both rows + would derive the same ID. +* **`PerfRun.Config` is stamped once per run.** `translate_results_to_kusto.sh` builds one + `--config-override` set from the queue-time `CFG_*` values and reuses it for both the baseline and + current rows. That is correct when the config genuinely is shared, but it means the two rows could + not record the differing switch values that are the entire point of the experiment. +* **Trend pollution.** No field marks a row as an experiment — `RunType` is already + `Sequential`/`Interweaved` — so the switch-on pass would be indistinguishable from an ordinary + measurement of the branch and would distort the very trends the other two pipelines exist to + protect. + +The comparison report and the raw BenchmarkDotNet artifacts are published as usual, and the build is +tagged `Switch ` so experiments are identifiable in the ADO build list. + +### Designing benchmarks for switch experiments + +A switch experiment flips behaviour on purpose, so a benchmark that measures that behaviour will +report a regression even when the change is working. `UseConnectionPoolV2` is the motivating example: +`ChannelDbConnectionPool` opens physical connections concurrently, where `WaitHandleDbConnectionPool` +serialises growth behind a `Semaphore(1, 1)`. `ConnectionPoolStressRunner` used to call +`ClearAllPools()` in `[IterationCleanup]`, so every iteration was a cold-start burst in which the +extra parallel opens had nothing to amortise against, and the runner reported the trade-off as a loss +because that was the only thing it could measure. + +That is a benchmark defect rather than something to explain away, so the runner now pre-warms the +pool to full capacity in `[GlobalSetup]` and no longer clears it between iterations. Establishing a +connection costs milliseconds while a pooled checkout costs microseconds, so any creation left in the +measured body swamps the pool cost the runner exists to measure. Keep that separation in mind when +adding a pool benchmark: warm the pool first unless connection establishment is precisely the thing +under test. + +The pipeline has no way to mark a result as acceptable, and deliberately so: a mute is only as good +as the reasoning behind it, and that reasoning belongs in the pull request where a reviewer can +challenge it. Prefer instead to fix the benchmark, or add one that measures the intended behaviour +directly. `ConnectionPoolRampRunner` was added for exactly this reason: it keeps the cold pool but +makes every caller *hold* its connection until all of them have connected, so the pool genuinely needs +N physical connections and the only variable left is how fast it can open them. That rewards +concurrent creation instead of penalising it, and it isolates cold start now that the stress runner no +longer conflates it with checkout cost. + +The same principle applies to how a benchmark schedules its workers. A sync `Open()` that has to +wait blocks whichever thread it runs on, so a pool whose waiter wake-up needs a queued continuation +stalls when every threadpool thread is already blocked; the wake-up waits on thread injection. On +the TFMs the perf project builds (net8.0-net10.0) the runtime is told about cooperative blocking and +compensates quickly, so that stall is tens to a few hundred milliseconds, and that is the only +expectation these benchmarks validate. On net462 the `Task` wait never notifies the pool, so the +wake-up falls to starvation detection and hill climbing and is materially slower; the pool carries no +framework guards and the driver still ships net462, so that path is live but unmeasured here. +Threadpool threads are the realistic case, because sync database +calls in ASP.NET run on them, and they are the only configuration in which that stall is visible. +Benchmarks therefore keep threadpool threads as the default and add dedicated-thread variants +alongside rather than instead: + +- `ConnectionPoolContentionRunner.SteadyStateOpenQueryCloseDedicatedThreads` runs the existing + workload on dedicated threads. A regression in both variants points at the pool; a regression in + only the threadpool variant points at the waiter wake path. +- `ConnectionPoolThreadPoolPressureRunner` pins the threadpool floor via `[Params]`, below the worker + count (starved) and above it (control), so the effect is reproducible instead of depending on + hill-climbing timing. + +This failure mode is tail latency, not a shifted median, so compare distributions. Aggregating with +a per-configuration minimum hides it completely. + +Note what those benchmarks are for. Saturating the thread pool with blocked synchronous calls is an +application configuration problem, not a pool defect: an application should keep its parallelism +below the thread pool's worker count so newly queued work still runs promptly, and pre-warming the +thread pool is the application's responsibility rather than the driver's. These benchmarks exist to +characterise where that boundary sits and to catch it moving, so a delta here is a prompt to check +the boundary has not shifted rather than a bug to fix. + +## Two-pass build model + +The `PerformanceTests` project references Microsoft.Data.SqlClient two ways, selected by MSBuild: + +- **Current** (default): `ProjectReference` to the in-repo source — the branch under test. +- **Baseline (package)**: `ReferenceType=Package` turns the reference into a `PackageReference`. Because the + repo uses **Central Package Management (CPM)**, the version is pinned with `VersionOverride` via + `-p:MdsPackageVersion=` (a plain `Version` is ignored under CPM). +- **Baseline (source)**: no reference switching at all — the baseline ref's own copy of the perf + project is built from `../sqlclient-perf-baseline-src`, keeping its default `ProjectReference` to + that ref's driver source. Used by the PR pipeline. +- **Baseline (switch experiment)**: no second build at all — `--switch-under-test` measures one + source tree twice, so the scripts build the `current` variant once and point both passes at it, + differing only in the runner config each pass is handed. Used by the experiment pipeline. + +The VM's `NuGet.config` exposes only the governed feed, and CPM rejects multiple unmapped sources +(`NU1507`). The baseline pass therefore restores through a **dedicated single-source config** +(`perf-baseline-nuget.config`, generated at runtime) pointing only at `https://api.nuget.org/v3/index.json`. + +### Benchmarks must compile against the oldest baseline + +The baseline pass compiles the **same** `PerformanceTests` sources against the *released* MDS +package, so a benchmark that calls an API introduced after `baselineVersion` fails that pass with +`CS1061`. The project defines cumulative `MDS_GE_` constants from `MdsPackageVersion` (see +`Microsoft.Data.SqlClient.PerformanceTests.csproj`); guard such calls and provide an older +fallback: + +```csharp +#if MDS_GE_6 + _ = reader.GetSqlJson(0).Value; // GetSqlJson was added in MDS 6.0 +#else + _ = reader.GetString(0); +#endif +``` + +Project mode, Package mode with no pinned version, and unparsable version strings define every +constant, since those builds reference a current MDS. Note that a fallback makes the baseline +measure a **different code path** than the current pass — the affected benchmark's delta is not +meaningful, so the runner should say so loudly at setup (see `JsonVsVarcharReadRunner`). + +## Comparison output + +`compare_perf.py` matches benchmarks by `(Type, Method, Parameters)` and reports, per benchmark, the +baseline/current mean (ms), mean %Δ, allocation %Δ, and a status (`regression` / `improvement` / +`unchanged` / `new` / `removed`). Outputs: + +- `results/comparison/comparison.md` (also copied to `results/summary.md`, which the template + attaches as the run summary), +- `results/comparison/comparison.json` (structured, for tooling). + +## Reducing noise + +The harness applies a set of harness-owned controls to reduce measurement noise. The lab already +supplies the isolated dedicated host, the tuned SQL instance, and the disjoint client CPU set +(`PERF_CLIENT_CPUS`); the run scripts add: + +| Control | What the harness does | +| ------- | --------------------- | +| Client CPU pin | Pins the benchmark process to `PERF_CLIENT_CPUS` (`taskset` on Linux, `ProcessorAffinity` on Windows). | +| Fail loud | Preflight `SELECT 1` before any pass, **and** a post-pass guard that fails the run if a pass produced **zero** benchmark results — so an empty comparison can never be reported green. | +| Warm-up | Touches the target DB in the preflight to warm the buffer pool / plan cache before the first measured benchmark. | +| Allocator tuning (Linux) | Exports `MALLOC_MMAP_THRESHOLD_=128MiB` and `MALLOC_TRIM_THRESHOLD_=-1` so large-buffer benches (`LargeDataRead`, `SqlBulkCopy`) stop re-`mmap`ing per iteration. | +| Network tuning (Linux) | Best-effort `sysctl` to widen the ephemeral port range and enable `tcp_tw_reuse` for churn benches (`ConnectionPoolStress`, `ConnectionPoolRamp`, `ConnectionPoolThreadPoolPressure`, `ParallelAsyncConnection`). Never fails the run. | +| Diagnostics | Writes `results/diagnostics/`: SQL instance config (MAXDOP, memory, affinity, tempdb files, `@@VERSION`), host CPU topology, and per-pass CPU-clock/thermal telemetry (before/after each pass). | +| Regression gate | `failOnRegression` threads `--fail-on-regression`; only a **candidate-slower** delta past the threshold fails, and in interleaved mode only after best-of-N confirmation. Default off. | +| Interleaving | In `interleaved` mode the harness runs **one benchmark unit at a time, baseline then candidate back-to-back**, so both sides see the same host state (see below). | +| Best-of-N confirmation | A unit flagged in the first interleaved pass is re-run `confirmationRuns` times; a regression is **confirmed** only on a strict majority. Unconfirmed flags are reported but never fail the gate. | + +### Interleaving + best-of-N (run model) + +`benchmarkRunMode` selects how the two variants are measured: + +- **`interleaved`** (default) — `interleave_perf.py` orchestrates the run. Both variants are built + **once** into separate output dirs (`perf-build-baseline`, `perf-build-current`), then for each + benchmark unit the baseline and candidate builds run **back-to-back** before moving to the next + unit. Because the same benchmark is measured on both sides within seconds, slow host drift affects + both roughly equally and cancels out of the delta. This relies on the `PerformanceTests` runner + supporting `PERF_LIST_BENCHMARKS` (enumerate enabled units) and `PERF_BENCHMARK=` (run a + single unit) — see `Program.cs`. + + After the first interleaved pass, only the units containing a flagged regression are re-run + `confirmationRuns` times (best-of-N). A regression is **confirmed** only when a strict majority of + the N passes agree `(count * 2 > N)`; otherwise it is reported as `regression (unconfirmed)` and + does **not** fail the `failOnRegression` gate. `confirmationRuns = 1` disables confirmation. + +- **`sequential`** — legacy model: the whole baseline suite runs, then the whole candidate suite, + then `compare_perf.py` diffs them. Kept as a fallback; produces the same `results/baseline`, + `results/current`, and `results/comparison/` layout so Kusto ingestion is identical. + +Both modes emit `results/comparison/comparison.md` + `comparison.json` and copy the markdown to +`results/summary.md`; interleaved mode adds a **Confirm** column and a confirmed/unconfirmed summary. + +### Further tuning (not yet implemented) + +- **Release-grade sampling / relaxed thresholds** — tune BenchmarkDotNet job counts and + significance thresholds in `runnerconfig.jsonc` / `BenchmarkConfig.cs` now that interleaving and + best-of-N are in place. + +## Kusto schema & ingestion + +Two tables: + +- **`PerfRun`** — one row per run (baseline OR current): `DerivedRunId` (PK = + `DriverName|CommitHash|PipelineRunId`), driver/machine/agent, `OperatingSystem` (`Windows`/`Linux`), + `Architecture` (`x64`/`x86`), `RunType` (`Sequential`/`Interweaved`), pipeline id + build URL, + branch + `BranchCategory`, `VersionString`, commit hash/date, `IsComparableBase`, `IngestedAt`. + `BranchCategory` buckets the ref as `main`, `release`, `dev`, `feature`, `pull_request` or + `other`. The internal ADO mirror's `internal/` prefix is stripped first, so `internal/main` and + `internal/release/*` land in the same buckets as their public counterparts. +- **`PerfBenchmarkResult`** — one row per benchmark: `BenchmarkId` (PK = + `DerivedRunId|BenchmarkName|MethodName|ParameterSignature`), timings in **milliseconds** + (BenchmarkDotNet reports nanoseconds; values are divided by 1,000,000), percentiles, throughput, + allocation, runtime/platform, and `DriverSpecificMetrics` (GC collections, lock contentions, …). + +The baseline and current passes share the pipeline run id and (for the current pass) the commit, so +to keep their `DerivedRunId`s distinct the baseline row uses `CommitHash = v` and +`IsComparableBase = true`; the current row uses the real commit and `IsComparableBase = false` with +the triggering branch name. + +Source = BenchmarkDotNet **JSON "full"** exporter files (`*-report-full.json`). The exporter is +enabled in `Config/BenchmarkConfig.cs` (`JsonExporter.Full`). + +### One-time database setup + +Before ingestion can run, create the two tables (`PerfRun`, `PerfBenchmarkResult`) in the target +database, with columns matching the schema summarized above. No server-side ingestion mappings need +to be created: `ingest_kusto.py` sends a **self-contained inline JSON column mapping** built from +each payload's own property names (which are identical to the table's column names), so ingestion +does not depend on any pre-created named mapping existing on the cluster. + +### Authentication + +Ingestion runs in an `AzureCLI@2` task using the ADO **ARM service connection** +(`KustoServiceConnection` from the `ADX Cluster Variables` group). That connection's **service +principal** must be granted, on the target database: + +- **Database Ingestor** — required to queue the ingestion, and +- **Database Viewer** — required for the post-ingestion verification queries. With Ingestor-only + rights the data still lands, but the verify step cannot read it back and logs a warning naming + this missing role. + +`ingest_kusto.py` authenticates to Kusto with `with_az_cli_authentication` (the service connection +is already `az login`'d inside the task) and performs a **queued** ingestion against the +data-management (`ingest-`) endpoint. + +### Running before the cluster exists + +Ingestion is **conditional**: it only runs when the `enableKustoIngestion` parameter is `true` (the +default) **and** `KustoClusterUri`, `KustoDatabase` and `KustoServiceConnection` (from the `ADX +Cluster Variables` group) are all non-empty. Set `enableKustoIngestion` to `false` to opt a run out +of ingestion explicitly. Until a cluster and service connection are +configured, the pipeline still runs both passes, produces the comparison, and publishes the +translated NDJSON as the `perf-kusto-payloads` artifact for manual/backfill ingestion. + +## Running the pipeline + +1. Open the performance test pipeline in Azure DevOps and select **Run pipeline**. +2. Choose the branch to benchmark; override `baselineVersion` only if needed. Ingestion is on by + default (`enableKustoIngestion`) and uses the `ADX Cluster Variables` group — populate + `KustoClusterUri` / `KustoDatabase` / `KustoServiceConnection` there to enable it, leave them empty + to skip ingestion, or untick **Ingest results into Kusto** to skip it for a single run. +3. After the run, review the **run summary** (comparison) and the `perf-results` / + `perf-kusto-payloads` artifacts. When a baseline pass ran, the build is tagged + **`Baseline `** so the baseline used is visible at a glance in the ADO build list. + +### Running the PR pipeline + +1. Open the **PR** performance test pipeline (`sqlclient-perf-pr-pipeline.yml`) in Azure DevOps and + select **Run pipeline**. +2. Choose the PR's branch to benchmark. Leave `baselineSourceRef` at `release/7.1` unless the PR + targets a different branch. No Kusto configuration is involved — PR results are never ingested. +3. After the run, review the **run summary** (comparison, labelled `@`) and the + `perf-results` artifact. The build is tagged **`Baseline `**. + +### Running the experiment pipeline + +1. Open the **experiment** performance test pipeline (`sqlclient-perf-experiment.yml`) in Azure + DevOps and select **Run pipeline**. +2. Choose the branch whose behaviour you want to measure — `main` to characterise the switch on its + own, or a PR branch to characterise it against that change — and pick `switchUnderTest`. There is + no baseline selector: the baseline *is* this branch with the switch off. No Kusto configuration is + involved; these results are never ingested (see [Why these runs are never ingested into + Kusto](#why-these-runs-are-never-ingested-into-kusto)). +3. After the run, review the **run summary** (comparison, labelled `=false`) and the + `perf-results` artifact. The build is tagged **`Switch `**. + +## Troubleshooting + +| Symptom | Likely cause / fix | +| ------- | ------------------ | +| `NU1507` during the baseline pass | Multiple NuGet sources under CPM. The baseline uses a single-source config; ensure `perf-baseline-nuget.config` is being passed via `-p:RestoreConfigFile`. | +| Baseline restore fails to find MDS | `baselineVersion` isn't a published NuGet.org version, or the VM has no outbound access to `api.nuget.org`. | +| Baseline pass fails to compile: `CS1061 ... does not contain a definition for ` | A benchmark calls an MDS API newer than `baselineVersion`. Guard it with the `MDS_GE_` constants and add an older fallback — see [Benchmarks must compile against the oldest baseline](#benchmarks-must-compile-against-the-oldest-baseline). | +| No comparison / summary | The baseline pass was skipped (empty `baselineVersion` / `baselineSourceRef`) or one pass produced no `*-report-full.json`. | +| `--switch-under-test is mutually exclusive with --baseline-version and --baseline-source-ref` | A switch experiment was combined with a source baseline. The experiment pipeline never does this; if you are invoking the scripts directly, clear the baseline selector — varying source and config at once makes the delta unattributable. | +| Switch experiment shows a ~0% delta everywhere | Expected for benchmarks the switch does not touch. If *every* benchmark is flat, check the run log's `Switch A/B` line actually names the switch, and that the switch is one the driver reads at startup via the runner config. | +| Baseline source ref not found (PR pipeline) | `baselineSourceRef` is not a branch on `origin`, and the fallback `git clone --branch ` of `baselineRepoUrl` also failed (ref does not exist there, or the VM has no outbound access to the remote). The reason git gave is echoed into the build log and saved to `/diagnostics/git-*.log`. | +| `Fetching baseline ref ... from the checkout's origin` is the last line, then the job stalls | Should no longer happen. The checkout is copied to the VM without credentials (ADO's checkout task defaults to `persistCredentials: false`), so a fetch from an authenticated `origin` used to sit on a `Username for ...` prompt forever. All network git calls now run with `GIT_TERMINAL_PROMPT=0`, no credential helper, stdin closed, and a `GIT_NET_TIMEOUT_SECS` (default 300s) hard timeout, so this fails in under a second and falls back to cloning `baselineRepoUrl`. A fetch failure here is expected and harmless on ADO. | +| Ingestion step skipped | `enableKustoIngestion` is `false`, or `KustoClusterUri`, `KustoDatabase` or `KustoServiceConnection` (from `ADX Cluster Variables`) is empty (expected until the cluster is provisioned). | +| Ingestion auth error | The service connection's SP lacks **Database Ingestor** on the target database. | +| "Kusto ingestion was queued, but the ingestion principal is not authorized to query the database" | The SP has **Database Ingestor** but not **Database Viewer**. Ingestion succeeded; grant **Database Viewer** so the verify step can confirm the rows landed. | +| `Kusto ingestion not yet queryable after Ns ... no ingestion failures were reported` (warning, step passes) | Expected, harmless: queued ingestion is asynchronous and small perf payloads can take longer than the verify window to become queryable. The step **warns and passes** because `.show ingestion failures` is clean, so the rows will land shortly. The step only **fails** when `.show ingestion failures` actually reports failures — in that case confirm the `PerfRun` / `PerfBenchmarkResult` tables exist with columns matching the schema above (a schema/column-name mismatch is the usual cause; the self-contained inline JSON mapping rules out a missing server-side named mapping). | +| Benchmarks not CPU-pinned | `PERF_CLIENT_CPUS` was not injected, or `taskset` is unavailable on the VM. | diff --git a/eng/pipelines/perf/scripts/compare_perf.py b/eng/pipelines/perf/scripts/compare_perf.py new file mode 100644 index 0000000000..609af7e8bd --- /dev/null +++ b/eng/pipelines/perf/scripts/compare_perf.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Compare two sets of BenchmarkDotNet "full" JSON reports and emit a delta. + +Reads every ``*-report-full.json`` under a baseline directory and a current +directory, matches benchmarks by (Type, Method, Parameters), and computes the +per-benchmark delta in mean execution time and allocated memory. + +Outputs: + * a GitHub-flavoured markdown table (``--out-md``), and + * a structured JSON document (``--out-json``) + +The script only uses the Python standard library so it can run on the perf VM +without installing any packages. +""" + +import argparse +import glob +import json +import os +import sys + + +NS_PER_MS = 1_000_000.0 + + +def _load_benchmarks(directory): + """Return {key: record} for every benchmark found under *directory*. + + key = "Type.Method(Parameters)". record carries the mean (ns), the + allocated bytes/op, and the display fields used for reporting. + """ + records = {} + pattern = os.path.join(directory, "**", "*-report-full.json") + for path in sorted(glob.glob(pattern, recursive=True)): + try: + with open(path, "r", encoding="utf-8-sig") as handle: + data = json.load(handle) + except (OSError, ValueError) as exc: + print(f"WARNING: could not parse {path}: {exc}", file=sys.stderr) + continue + + for bench in data.get("Benchmarks", []): + btype = bench.get("Type", "") + method = bench.get("Method", "") + params = bench.get("Parameters", "") or "" + stats = bench.get("Statistics") or {} + mean_ns = stats.get("Mean") + if mean_ns is None: + continue + memory = bench.get("Memory") or {} + alloc = memory.get("BytesAllocatedPerOperation") + + key = f"{btype}.{method}({params})" + records[key] = { + "benchmarkName": btype, + "methodName": method, + "parameterSignature": params, + "meanNs": float(mean_ns), + "allocatedBytes": float(alloc) if alloc is not None else None, + } + return records + + +def _pct(baseline, current): + if baseline is None or current is None: + return None + if baseline == 0: + # A percentage change from a zero baseline is undefined (0 -> 0 is no change; 0 -> X is an + # infinite increase). Return 0.0 only for the genuine no-change case; otherwise None, and let + # callers surface the raw before/after values so a 0 -> X regression is still visible. + return 0.0 if current == 0 else None + return (current - baseline) / baseline * 100.0 + + +def build_comparison(baseline_dir, current_dir, threshold_pct): + baseline = _load_benchmarks(baseline_dir) + current = _load_benchmarks(current_dir) + + entries = [] + for key in sorted(set(baseline) | set(current)): + b = baseline.get(key) + c = current.get(key) + ref = c or b + entry = { + "key": key, + "benchmarkName": ref["benchmarkName"], + "methodName": ref["methodName"], + "parameterSignature": ref["parameterSignature"], + "baselineMeanMs": (b["meanNs"] / NS_PER_MS) if b else None, + "currentMeanMs": (c["meanNs"] / NS_PER_MS) if c else None, + "baselineAllocBytes": b["allocatedBytes"] if b else None, + "currentAllocBytes": c["allocatedBytes"] if c else None, + } + + if b and c: + entry["meanDeltaPct"] = _pct(b["meanNs"], c["meanNs"]) + entry["meanRatio"] = (c["meanNs"] / b["meanNs"]) if b["meanNs"] else None + # Compute the allocation delta whenever both sides report a value. A 0-byte baseline is + # valid, so gate on 'is not None' rather than truthiness -- gating on truthiness would drop + # a real 0 -> X allocation regression. The percentage itself is undefined for a 0 baseline + # (see _pct); the raw baseline/current byte counts on the entry keep 0 -> X visible. + if b["allocatedBytes"] is not None and c["allocatedBytes"] is not None: + entry["allocDeltaPct"] = _pct(b["allocatedBytes"], c["allocatedBytes"]) + else: + entry["allocDeltaPct"] = None + delta = entry["meanDeltaPct"] + if delta is None: + entry["status"] = "unknown" + elif delta > threshold_pct: + entry["status"] = "regression" + elif delta < -threshold_pct: + entry["status"] = "improvement" + else: + entry["status"] = "unchanged" + elif c and not b: + entry["meanDeltaPct"] = None + entry["meanRatio"] = None + entry["allocDeltaPct"] = None + entry["status"] = "current-only" + else: + entry["meanDeltaPct"] = None + entry["meanRatio"] = None + entry["allocDeltaPct"] = None + entry["status"] = "baseline-only" + + entries.append(entry) + + # Sort worst-regression first, then by name for stability. + def _sort_key(e): + d = e["meanDeltaPct"] + return (-(d if d is not None else -1e18), e["key"]) + + entries.sort(key=_sort_key) + return entries + + +def _fmt_ms(value): + return f"{value:.4f}" if value is not None else "-" + + +def _fmt_pct(value): + if value is None: + return "-" + return f"{value:+.2f}%" + + +def _fmt_bytes(value): + return f"{int(value)}" if value is not None else "-" + + +def _fmt_alloc(entry): + """Alloc column: a percentage when it is defined, otherwise the raw byte transition so a + 0 -> X regression (undefined as a percentage) is still shown rather than collapsing to '-'.""" + pct = entry.get("allocDeltaPct") + if pct is not None: + return f"{pct:+.2f}%" + base = entry.get("baselineAllocBytes") + cur = entry.get("currentAllocBytes") + if base is not None and cur is not None and base != cur: + return f"{int(base)} → {int(cur)} B" + return "-" + + +def render_markdown(entries, baseline_version, threshold_pct): + regressions = [e for e in entries if e["status"] == "regression"] + improvements = [e for e in entries if e["status"] == "improvement"] + + lines = [] + lines.append("# SqlClient Performance Comparison") + lines.append("") + lines.append(f"Baseline: **{baseline_version}**  |  " + f"Regression threshold: **{threshold_pct:.0f}%**") + lines.append("") + lines.append(f"- Benchmarks compared: **{len(entries)}**") + lines.append(f"- Regressions (slower > {threshold_pct:.0f}%): **{len(regressions)}**") + lines.append(f"- Improvements (faster > {threshold_pct:.0f}%): **{len(improvements)}**") + lines.append("") + lines.append("| Status | Benchmark | Method | Params | Baseline (ms) | " + "Current (ms) | Mean Δ | Alloc Δ |") + lines.append("| ------ | --------- | ------ | ------ | ------------- | " + "------------ | ------ | ------- |") + + icon = { + "regression": "🔴 regression", + "improvement": "🟢 improvement", + "unchanged": "⚪ unchanged", + "current-only": "🆕 new", + "baseline-only": "➖ removed", + "unknown": "❔", + } + for e in entries: + lines.append( + "| {status} | {name} | {method} | {params} | {base} | {cur} | " + "{delta} | {alloc} |".format( + status=icon.get(e["status"], e["status"]), + name=e["benchmarkName"], + method=e["methodName"], + params=e["parameterSignature"] or "-", + base=_fmt_ms(e["baselineMeanMs"]), + cur=_fmt_ms(e["currentMeanMs"]), + delta=_fmt_pct(e["meanDeltaPct"]), + alloc=_fmt_alloc(e), + ) + ) + lines.append("") + return "\n".join(lines) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline-dir", required=True) + parser.add_argument("--current-dir", required=True) + parser.add_argument("--out-md", required=True) + parser.add_argument("--out-json", required=True) + parser.add_argument("--baseline-version", default="baseline") + parser.add_argument("--threshold", type=float, default=10.0, + help="Percent slowdown that counts as a regression.") + parser.add_argument("--fail-on-regression", action="store_true", + help="Exit non-zero if any regression is detected.") + args = parser.parse_args(argv) + + entries = build_comparison(args.baseline_dir, args.current_dir, args.threshold) + + os.makedirs(os.path.dirname(os.path.abspath(args.out_md)), exist_ok=True) + os.makedirs(os.path.dirname(os.path.abspath(args.out_json)), exist_ok=True) + + markdown = render_markdown(entries, args.baseline_version, args.threshold) + with open(args.out_md, "w", encoding="utf-8") as handle: + handle.write(markdown + "\n") + + with open(args.out_json, "w", encoding="utf-8") as handle: + json.dump( + { + "baselineVersion": args.baseline_version, + "thresholdPct": args.threshold, + "entries": entries, + }, + handle, + indent=2, + ) + + regressions = [e for e in entries if e["status"] == "regression"] + print(f"Compared {len(entries)} benchmarks; {len(regressions)} regression(s).") + print(f"Wrote {args.out_md} and {args.out_json}.") + + if args.fail_on_regression and regressions: + print("Regressions detected; failing as requested.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/ingest_kusto.py b/eng/pipelines/perf/scripts/ingest_kusto.py new file mode 100644 index 0000000000..fb0b9d96d0 --- /dev/null +++ b/eng/pipelines/perf/scripts/ingest_kusto.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Ingest translated perf-results NDJSON files into Azure Data Explorer (Kusto). + +Runs on the pipeline agent inside an ``AzureCLI@2`` task so that the Azure DevOps +service connection's service principal is already logged in via ``az``. The +script therefore authenticates to Kusto with *az CLI* credentials and performs a +queued ingestion of the PerfRun and PerfBenchmarkResult NDJSON files produced by +``perf_to_kusto.py``. + +Queued ingestion is asynchronous: ``ingest_from_file`` only *enqueues* the data +and returns success even when the data later fails to land (e.g. a schema +mismatch, or a *missing* server-side ingestion mapping - which the data-management +service silently retries for ~2 days before recording a failure). Those failures +surface only in Kusto's ingestion failure system, so the pipeline step used to go +green (or hang) while the database stayed empty. To avoid that this script: + + * sends a **self-contained inline JSON column mapping** (built from the payload's + own property names, which match the target table's columns) instead of + referencing a named server-side mapping that must be pre-created on the cluster, + * sets ``flush_immediately`` so the data-management service seals the batch + right away instead of waiting out the (default 5 minute) batching window, and + * verifies, after queuing, that the expected rows actually became queryable -- + polling the target tables and, on timeout, printing ``.show ingestion + failures`` so the real error is visible in the build log (and failing the + step). + +Requires the ``azure-kusto-ingest`` package (``pip install azure-kusto-data +azure-kusto-ingest``); the pipeline step installs it before invoking this script. +""" + +import argparse +import json +import os +import sys +import time + + +def _engine_uri(cluster): + """Return the engine (query) endpoint, stripping any 'ingest-' prefix.""" + if "://" in cluster: + scheme, rest = cluster.split("://", 1) + if rest.startswith("ingest-"): + rest = rest[len("ingest-"):] + return f"{scheme}://{rest}" + return cluster + + +def _ingest_uri(cluster): + """Return the data-management (ingest) endpoint used by queued ingestion.""" + if "://" in cluster: + scheme, rest = cluster.split("://", 1) + if not rest.startswith("ingest-"): + return f"{scheme}://ingest-{rest}" + return cluster + + +def _ndjson_columns(data_file): + """Ordered union of the JSON property names across every record in an NDJSON file. + + The translated payload uses property names that are identical to the target table's + column names, so this doubles as the set of columns to map.""" + columns = [] + seen = set() + with open(data_file, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except ValueError: + continue + for key in obj: + if key not in seen: + seen.add(key) + columns.append(key) + return columns + + +def _inline_json_mapping(data_file): + """Build an inline JSON column mapping (column -> $.column) from the payload's own keys. + + Ingestion deliberately does NOT reference a pre-created server-side named mapping: relying on + one silently breaks ingestion whenever the mapping is missing from the cluster (the queued + request is retried for ~2 days before a failure is even recorded). Because the NDJSON property + names match the table column names 1:1, a self-contained inline mapping removes that external + dependency entirely.""" + from azure.kusto.ingest import ColumnMapping + + # An empty column_type lets Kusto use each table column's declared type (works for string, + # numeric, datetime, bool and dynamic columns alike). + return [ColumnMapping(column_name=col, column_type="", path=f"$.{col}") + for col in _ndjson_columns(data_file)] + + +def _ingest(cluster, database, table, data_file): + from azure.kusto.data import KustoConnectionStringBuilder + from azure.kusto.ingest import ( + QueuedIngestClient, + IngestionProperties, + FileDescriptor, + ) + from azure.kusto.data.data_format import DataFormat, IngestionMappingKind + from azure.kusto.ingest import ReportLevel + + if not os.path.exists(data_file) or os.path.getsize(data_file) == 0: + print(f"Skipping {table}: '{data_file}' is missing or empty.") + return 0 + + column_mappings = _inline_json_mapping(data_file) + if not column_mappings: + print(f"Skipping {table}: '{data_file}' has no recognizable JSON records.") + return 0 + + kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(_ingest_uri(cluster)) + client = QueuedIngestClient(kcsb) + + props = IngestionProperties( + database=database, + table=table, + data_format=DataFormat.MULTIJSON, + column_mappings=column_mappings, + ingestion_mapping_kind=IngestionMappingKind.JSON, + report_level=ReportLevel.FailuresAndSuccesses, + # Seal the batch immediately: perf payloads are tiny and we want the rows + # queryable in seconds (and any verification to run promptly), not after + # the default multi-minute batching window. + flush_immediately=True, + ) + + descriptor = FileDescriptor(data_file, os.path.getsize(data_file)) + client.ingest_from_file(descriptor, ingestion_properties=props) + print(f"Queued ingestion of '{data_file}' -> {database}.{table} " + f"(inline JSON mapping, {len(column_mappings)} columns).") + return 0 + + +def _count_rows(path): + """Number of non-empty NDJSON lines in a file (0 when missing).""" + if not os.path.exists(path): + return 0 + with open(path, "r", encoding="utf-8") as fh: + return sum(1 for line in fh if line.strip()) + + +def _pipeline_ids(paths): + """Collect the distinct PipelineRunId values from PerfRun NDJSON files.""" + ids = set() + for path in paths: + if not os.path.exists(path): + continue + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + pid = json.loads(line).get("PipelineRunId") + except ValueError: + pid = None + if pid: + ids.add(str(pid)) + return ids + + +def _scalar(client, database, query): + resp = client.execute(database, query) + return int(resp.primary_results[0].rows[0][0]) + + +def _is_authorization_error(exc): + """True when an exception looks like a Kusto authorization/permission denial + (as opposed to a transient network error), so the operator can be told to + grant the querying role rather than chase a connectivity problem.""" + if exc is None: + return False + text = str(exc).lower() + needles = ( + "forbidden", "unauthorized", "not authorized", "does not have permission", + "principal", "403", "e_access", "access denied", + ) + return any(n in text for n in needles) + + +def _dump_failures(client, database): + """Print any recent ingestion failures for the perf tables. + + Returns the number of failures found, or ``None`` when the failure query + itself could not run (e.g. the principal lacks the monitoring role). The + caller uses this to distinguish a genuine ingestion failure (fail the step) + from data that is merely still flushing (do not fail the step).""" + cmd = (".show ingestion failures " + "| where Table in ('PerfRun', 'PerfBenchmarkResult') " + "| where FailedOn > ago(2h) " + "| project FailedOn, Table, ErrorCode, FailureKind, Details, OperationId " + "| order by FailedOn desc | take 25") + try: + resp = client.execute_mgmt(database, cmd) + rows = resp.primary_results[0].rows + if not rows: + print("No recent ingestion failures reported by Kusto " + "(data is still flushing and will land asynchronously).", + file=sys.stderr) + return 0 + print("Recent Kusto ingestion failures:", file=sys.stderr) + for r in rows: + print(f" {r[0]} {r[1]} [{r[2]}/{r[3]}] {r[4]} (op {r[5]})", + file=sys.stderr) + return len(rows) + except Exception as exc: # noqa: BLE001 - diagnostics best-effort + print(f" (could not query ingestion failures: {exc})", file=sys.stderr) + return None + + +def _verify(cluster, database, run_table, results_table, pipeline_ids, + expected_run, expected_results, timeout_s, interval_s): + """Poll the target tables until the expected rows appear. + + Returns 0 on success, 1 when rows are provably missing after the timeout. + When verification queries cannot run at all (e.g. the ingestion principal + lacks query rights) this is best-effort: it warns and returns 0 so the step + is not failed on a permission gap, since the ingestion itself was queued. + """ + if not pipeline_ids or (expected_run == 0 and expected_results == 0): + print("Nothing to verify (no rows were queued).") + return 0 + + from azure.kusto.data import KustoClient, KustoConnectionStringBuilder + + client = KustoClient( + KustoConnectionStringBuilder.with_az_cli_authentication(_engine_uri(cluster))) + + id_list = ", ".join(f'"{i}"' for i in sorted(pipeline_ids)) + run_query = f"{run_table} | where PipelineRunId in ({id_list}) | count" + # PerfBenchmarkResult has no PipelineRunId column, but DerivedRunId embeds it + # as the trailing '|' segment. + res_conds = " or ".join(f'DerivedRunId endswith "|{i}"' for i in sorted(pipeline_ids)) + res_query = f"{results_table} | where {res_conds} | count" + + print(f"Verifying ingestion for PipelineRunId in [{id_list}] " + f"(expecting {run_table}>={expected_run}, {results_table}>={expected_results}); " + f"timeout {timeout_s}s ...") + + deadline = time.time() + timeout_s + query_ever_ok = False + last_exc = None + run_have = res_have = -1 + while True: + try: + run_have = _scalar(client, database, run_query) + res_have = _scalar(client, database, res_query) + query_ever_ok = True + except Exception as exc: # noqa: BLE001 - transient/permission, retried + last_exc = exc + print(f" (verification query failed, will retry: {exc})") + + if query_ever_ok: + print(f" {run_table}={run_have}/{expected_run}, " + f"{results_table}={res_have}/{expected_results}") + if run_have >= expected_run and res_have >= expected_results: + print("Verified: all expected rows are queryable in Kusto.") + return 0 + + if time.time() >= deadline: + break + time.sleep(interval_s) + + if not query_ever_ok: + detail = str(last_exc) if last_exc is not None else "no further detail" + if _is_authorization_error(last_exc): + # Queued ingestion only needs the 'Database Ingestor' role; verification *queries* the + # data back and additionally needs 'Database Viewer'. A principal with Ingestor-only + # rights ingests fine but cannot verify, so name the missing role instead of blaming the + # network. The ingestion itself was queued, so this stays a warning (exit 0). + print("##vso[task.logissue type=warning]Kusto ingestion was queued, but the ingestion " + "principal is not authorized to query the database, so ingestion could not be " + "verified. Grant the service connection's service principal the 'Database Viewer' " + "role on this database (in addition to 'Database Ingestor'). " + f"Details: {detail}", file=sys.stderr) + else: + print("##vso[task.logissue type=warning]Could not verify Kusto ingestion " + "(query endpoint unreachable or the verification query failed). Data was queued; " + f"check the database manually. Details: {detail}", file=sys.stderr) + return 0 + + # The verification loop timed out with rows still missing. Queued ingestion is + # asynchronous: small perf payloads routinely take longer than the polling window + # to become queryable even though they ingest successfully (observed to land + # anywhere from ~30s to several minutes later). Only fail the step when Kusto + # actually reports ingestion failures; otherwise the data is still flushing and + # failing here would be a false negative. + failures = _dump_failures(client, database) + if failures: + print(f"##vso[task.logissue type=error]Kusto ingestion failed: " + f"{run_table}={run_have}/{expected_run}, " + f"{results_table}={res_have}/{expected_results} after {timeout_s}s; " + f"{failures} ingestion failure(s) reported (see above).", + file=sys.stderr) + return 1 + + print(f"##vso[task.logissue type=warning]Kusto ingestion not yet queryable after " + f"{timeout_s}s ({run_table}={run_have}/{expected_run}, " + f"{results_table}={res_have}/{expected_results}), but no ingestion failures were " + f"reported. Queued ingestion completes asynchronously, so the rows will land " + f"shortly; not failing the step.", file=sys.stderr) + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cluster", required=True, + help="Kusto cluster URI, e.g. https://..kusto.windows.net") + parser.add_argument("--database", required=True) + parser.add_argument("--run-file", required=True, + help="PerfRun NDJSON file(s), comma-separated.") + parser.add_argument("--results-file", required=True, + help="PerfBenchmarkResult NDJSON file(s), comma-separated.") + parser.add_argument("--run-table", default="PerfRun") + parser.add_argument("--results-table", default="PerfBenchmarkResult") + parser.add_argument("--verify-timeout", type=int, + default=int(os.environ.get("KUSTO_VERIFY_TIMEOUT", "600")), + help="Seconds to wait for ingested rows to become queryable " + "(0 disables verification).") + parser.add_argument("--verify-interval", type=int, + default=int(os.environ.get("KUSTO_VERIFY_INTERVAL", "15")), + help="Polling interval in seconds for verification.") + args = parser.parse_args(argv) + + try: + import azure.kusto.ingest # noqa: F401 + except ImportError: + print("ERROR: azure-kusto-ingest is not installed. " + "Run 'pip install azure-kusto-data azure-kusto-ingest'.", + file=sys.stderr) + return 2 + + run_files = [f.strip() for f in args.run_file.split(",") if f.strip()] + results_files = [f.strip() for f in args.results_file.split(",") if f.strip()] + + expected_run = sum(_count_rows(f) for f in run_files) + expected_results = sum(_count_rows(f) for f in results_files) + pipeline_ids = _pipeline_ids(run_files) + + for data_file in run_files: + _ingest(args.cluster, args.database, args.run_table, data_file) + for data_file in results_files: + _ingest(args.cluster, args.database, args.results_table, data_file) + + print("Ingestion requests submitted (queued).") + + if args.verify_timeout <= 0: + print("Verification disabled (--verify-timeout <= 0).") + return 0 + + return _verify(args.cluster, args.database, args.run_table, args.results_table, + pipeline_ids, expected_run, expected_results, + args.verify_timeout, args.verify_interval) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/interleave_perf.py b/eng/pipelines/perf/scripts/interleave_perf.py new file mode 100644 index 0000000000..6522adced1 --- /dev/null +++ b/eng/pipelines/perf/scripts/interleave_perf.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""Interleaved, best-of-N benchmark orchestrator for the SqlClient perf pipeline. + +Implements the two structural noise-reduction controls from InternalDriverTools +wiki 339 ("Reducing Noise in Performance Tests"): + +* **Interleaving (§2.2 / §2.3)** — instead of running the whole baseline suite and + then the whole candidate suite (so a given benchmark is measured tens of minutes + apart), this runs *one benchmark unit at a time*, executing the baseline build and + the candidate build back-to-back for that unit before moving on. Any slow host + drift then affects both sides of each comparison roughly equally. + +* **Best-of-N confirmation (§2.6)** — a single measurement can flag a regression + purely from noise. After the first interleaved pass, only the units that showed a + regression are re-run N-1 more times (still interleaved); a regression is + *confirmed* only when a strict majority of the N passes agree. This is what makes + the ``--fail-on-regression`` gate trustworthy. + +The benchmark executable is the ``PerformanceTests`` app built two ways (same test +sources, different Microsoft.Data.SqlClient reference). It understands two env vars +added for this orchestrator: + +* ``PERF_LIST_BENCHMARKS`` — print the enabled unit names and exit. +* ``PERF_BENCHMARK=`` — run only that unit. + +Only the Python standard library is used so it runs on the perf VM unchanged. +""" + +import argparse +import glob +import json +import os +import shutil +import subprocess +import sys + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPT_DIR) +import compare_perf # noqa: E402 (local module, sits next to this script) + + +# -------------------------------------------------------------------------------------------------- +# CPU affinity (client pinning, wiki §2.4) — best effort, cross platform. +# -------------------------------------------------------------------------------------------------- +def parse_cpus(spec): + """Parse a CPU spec like "16-31" or "0,2,4" or "0-3,8" into a sorted int list. + + CPU pinning is a best-effort optimisation, never a gate, so a malformed spec must not + fail the run: on any unparseable segment this warns and returns [] (i.e. no pinning).""" + if not spec: + return [] + cpus = set() + try: + for part in spec.split(","): + part = part.strip() + if not part: + continue + if "-" in part: + lo, hi = part.split("-", 1) + cpus.update(range(int(lo), int(hi) + 1)) + else: + cpus.add(int(part)) + except ValueError: + print(f"WARNING: could not parse CPU spec {spec!r}; running without CPU pinning.", + file=sys.stderr) + return [] + return sorted(cpus) + + +def apply_affinity(proc, cpus): + """Pin *proc* to *cpus*. Never raises — pinning is an optimisation, not a gate.""" + if not cpus: + return + try: + if hasattr(os, "sched_setaffinity"): # Linux + os.sched_setaffinity(proc.pid, set(cpus)) + return + if os.name == "nt": # Windows + import ctypes + + # SetProcessAffinityMask takes a single-word mask that only addresses CPUs 0-63; + # higher indices require processor-group APIs. Rather than set a mask that would + # silently pin to the wrong CPUs, skip pinning with a warning. + if any(c >= 64 for c in cpus): + print(f"WARNING: CPU index >= 64 in {cpus}; SetProcessAffinityMask cannot address " + f"processor groups, so pid {proc.pid} runs without CPU pinning.", + file=sys.stderr) + return + mask = 0 + for c in cpus: + mask |= (1 << c) + handle = int(proc._handle) # noqa: SLF001 (Popen exposes the OS handle here) + if ctypes.windll.kernel32.SetProcessAffinityMask(handle, ctypes.c_size_t(mask)) == 0: + print(f"WARNING: SetProcessAffinityMask failed for pid {proc.pid}.", file=sys.stderr) + except Exception as exc: # noqa: BLE001 + print(f"WARNING: could not pin pid {getattr(proc, 'pid', '?')} to CPUs {cpus}: {exc}", + file=sys.stderr) + + +# -------------------------------------------------------------------------------------------------- +# Running one unit and collecting its artifacts. +# -------------------------------------------------------------------------------------------------- +def run_unit_process(exe_dir, assembly, unit, cwd, cpus, log_path, env_overrides=None): + """Run one benchmark *unit* from the build at *exe_dir* in *cwd*. + + Returns the subprocess return code. Kept as a small seam so tests can substitute + a fake runner. *env_overrides*, when given, is applied on top of the inherited + environment (e.g. a per-variant RUNNER_CONFIG so baseline and current can run with + different SqlClient behaviour flags, such as comparing the legacy vs new connection + pool from the SAME build). + """ + os.makedirs(cwd, exist_ok=True) + cmd = ["dotnet", os.path.join(exe_dir, assembly)] + env = dict(os.environ) + env["PERF_BENCHMARK"] = unit + env.pop("PERF_LIST_BENCHMARKS", None) + if env_overrides: + env.update(env_overrides) + + with open(log_path, "w", encoding="utf-8") as log: + proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log, + stderr=subprocess.STDOUT) + apply_affinity(proc, cpus) + return proc.wait() + + +def _report_files(root): + return sorted(glob.glob(os.path.join(root, "**", "*-report-full.json"), recursive=True)) + + +def collect_results(cwd, dest): + """Copy the BenchmarkDotNet reports produced under *cwd* into *dest*. + + Returns the set of benchmark ``Type`` names found (used to map a unit to the + report rows it produced, so best-of-N can re-run the right unit for a flagged key). + """ + os.makedirs(dest, exist_ok=True) + types = set() + artifacts = os.path.join(cwd, "BenchmarkDotNet.Artifacts") + for path in _report_files(artifacts): + shutil.copy2(path, os.path.join(dest, os.path.basename(path))) + try: + with open(path, "r", encoding="utf-8-sig") as fh: + for bench in json.load(fh).get("Benchmarks", []): + if bench.get("Type"): + types.add(bench["Type"]) + except (OSError, ValueError): + pass + # Also keep the human-readable GitHub markdown reports alongside, best effort. + for md in glob.glob(os.path.join(artifacts, "**", "*-report-github.md"), recursive=True): + shutil.copy2(md, os.path.join(dest, os.path.basename(md))) + return types + + +# -------------------------------------------------------------------------------------------------- +# Interleaved passes. +# -------------------------------------------------------------------------------------------------- +class Runner: + """Holds the invariant run parameters and performs interleaved unit passes.""" + + def __init__(self, baseline_dir, current_dir, assembly, work_dir, cpus, + baseline_runner_config=None, current_runner_config=None): + self.baseline_dir = baseline_dir + self.current_dir = current_dir + self.assembly = assembly + self.work_dir = work_dir + self.cpus = cpus + # Optional per-variant RUNNER_CONFIG override (e.g. --switch-under-test needs baseline + # and current to run with different SqlClient behaviour flags even though they share the + # same build). None means "no override" -> both variants use the ambient RUNNER_CONFIG. + self.baseline_runner_config = baseline_runner_config + self.current_runner_config = current_runner_config + + def list_units(self): + cmd = ["dotnet", os.path.join(self.current_dir, self.assembly)] + env = dict(os.environ) + env["PERF_LIST_BENCHMARKS"] = "1" + env.pop("PERF_BENCHMARK", None) + out = subprocess.check_output(cmd, cwd=self.work_dir, env=env, text=True) + return [line.strip() for line in out.splitlines() if line.strip()] + + def _run_one(self, variant, exe_dir, unit, rep, agg_dir): + cwd = os.path.join(self.work_dir, f"rep{rep}", variant, unit) + if os.path.isdir(cwd): + shutil.rmtree(cwd) + os.makedirs(cwd, exist_ok=True) + log_path = os.path.join(cwd, "run.log") + runner_config = (self.baseline_runner_config if variant == "baseline" + else self.current_runner_config) + env_overrides = {"RUNNER_CONFIG": runner_config} if runner_config else None + rc = run_unit_process(exe_dir, self.assembly, unit, cwd, self.cpus, log_path, + env_overrides=env_overrides) + if rc != 0: + _tail(log_path) + raise RuntimeError(f"benchmark unit '{unit}' ({variant}, rep {rep}) failed (exit {rc}).") + types = collect_results(cwd, agg_dir) + if not types: + _tail(log_path) + raise RuntimeError( + f"benchmark unit '{unit}' ({variant}, rep {rep}) produced no results " + f"(a broken benchmark must not be reported as a pass).") + return types + + def interleave(self, units, rep, baseline_agg, current_agg): + """Run each unit baseline-then-candidate; aggregate reports per variant. + + Returns a dict mapping unit name -> set of report Type names it produced. + """ + unit_types = {} + for unit in units: + print(f"[rep {rep}] {unit}: baseline -> candidate", flush=True) + # Interleaved: measure baseline and candidate for this unit back-to-back. + self._run_one("baseline", self.baseline_dir, unit, rep, baseline_agg) + types = self._run_one("current", self.current_dir, unit, rep, current_agg) + unit_types[unit] = types + return unit_types + + +def _tail(path, n=40): + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + lines = fh.readlines() + sys.stderr.write("".join(lines[-n:])) + except OSError: + pass + + +# -------------------------------------------------------------------------------------------------- +# Comparison + confirmation. +# -------------------------------------------------------------------------------------------------- +def _regression_keys(baseline_dir, current_dir, threshold): + entries = compare_perf.build_comparison(baseline_dir, current_dir, threshold) + return entries, {e["key"] for e in entries if e["status"] == "regression"} + + +def orchestrate(runner, units, results_dir, threshold, reps): + baseline_agg = os.path.join(results_dir, "baseline") + current_agg = os.path.join(results_dir, "current") + comparison_dir = os.path.join(results_dir, "comparison") + os.makedirs(comparison_dir, exist_ok=True) + + # --- Pass 1: interleave every enabled unit ----------------------------------------------------- + unit_types = runner.interleave(units, 1, baseline_agg, current_agg) + type_to_unit = {} + for unit, types in unit_types.items(): + for t in types: + type_to_unit[t] = unit + + entries, reg_keys_1 = _regression_keys(baseline_agg, current_agg, threshold) + + # Which units contain a rep-1 regression? Those are the best-of-N candidates. + candidate_units = [] + for e in entries: + if e["status"] == "regression": + unit = type_to_unit.get(e["benchmarkName"]) + if unit and unit not in candidate_units: + candidate_units.append(unit) + + # Per-key regression tally across reps (rep 1 counts once). + reg_counts = {k: 1 for k in reg_keys_1} + + # --- Passes 2..N: re-run only the candidate units, still interleaved --------------------------- + if reps > 1 and candidate_units: + print(f"Best-of-{reps}: confirming {len(candidate_units)} candidate unit(s): " + f"{', '.join(candidate_units)}", flush=True) + for rep in range(2, reps + 1): + b_dir = os.path.join(results_dir, "reps", f"rep{rep}", "baseline") + c_dir = os.path.join(results_dir, "reps", f"rep{rep}", "current") + runner.interleave(candidate_units, rep, b_dir, c_dir) + _, reg_keys_r = _regression_keys(b_dir, c_dir, threshold) + for k in reg_keys_1: + if k in reg_keys_r: + reg_counts[k] = reg_counts.get(k, 0) + 1 + + # --- Confirmation verdict (strict majority of N) ----------------------------------------------- + total_reps = reps + for e in entries: + key = e["key"] + if e["status"] == "regression": + count = reg_counts.get(key, 0) + confirmed = (count * 2) > total_reps + e["regressionReps"] = count + e["totalReps"] = total_reps + e["confirmedRegression"] = confirmed + if not confirmed: + e["status"] = "regression-unconfirmed" + else: + e["regressionReps"] = 0 + e["totalReps"] = total_reps + e["confirmedRegression"] = False + + confirmed = [e for e in entries if e.get("confirmedRegression")] + unconfirmed = [e for e in entries if e["status"] == "regression-unconfirmed"] + return entries, confirmed, unconfirmed + + +# -------------------------------------------------------------------------------------------------- +# Rendering. +# -------------------------------------------------------------------------------------------------- +_ICON = { + "regression": "🔴 regression", + "regression-unconfirmed": "🟠 regression (unconfirmed)", + "improvement": "🟢 improvement", + "unchanged": "⚪ unchanged", + "current-only": "🆕 new", + "baseline-only": "➖ removed", + "unknown": "❔", +} + + +def render_markdown(entries, confirmed, unconfirmed, baseline_version, threshold, reps): + improvements = [e for e in entries if e["status"] == "improvement"] + lines = [] + lines.append("# SqlClient Performance Comparison (interleaved, best-of-N)") + lines.append("") + lines.append(f"Baseline: **{baseline_version}**  |  " + f"Regression threshold: **{threshold:.0f}%**  |  " + f"Confirmation runs: **N = {reps}**") + lines.append("") + lines.append(f"- Benchmarks compared: **{len(entries)}**") + lines.append(f"- **Confirmed** regressions (majority of {reps}): **{len(confirmed)}**") + lines.append(f"- Unconfirmed (flagged once, not reproduced): **{len(unconfirmed)}**") + lines.append(f"- Improvements (faster > {threshold:.0f}%): **{len(improvements)}**") + lines.append("") + lines.append("| Status | Benchmark | Method | Params | Baseline (ms) | " + "Current (ms) | Mean Δ | Alloc Δ | Confirm |") + lines.append("| ------ | --------- | ------ | ------ | ------------- | " + "------------ | ------ | ------- | ------- |") + for e in entries: + if e["totalReps"] > 1 and (e["confirmedRegression"] or e["status"] == "regression-unconfirmed"): + confirm = f"{e['regressionReps']}/{e['totalReps']}" + else: + confirm = "-" + lines.append( + "| {status} | {name} | {method} | {params} | {base} | {cur} | " + "{delta} | {alloc} | {confirm} |".format( + status=_ICON.get(e["status"], e["status"]), + name=e["benchmarkName"], + method=e["methodName"], + params=e["parameterSignature"] or "-", + base=compare_perf._fmt_ms(e["baselineMeanMs"]), + cur=compare_perf._fmt_ms(e["currentMeanMs"]), + delta=compare_perf._fmt_pct(e["meanDeltaPct"]), + alloc=compare_perf._fmt_pct(e["allocDeltaPct"]), + confirm=confirm, + ) + ) + lines.append("") + return "\n".join(lines) + + +# -------------------------------------------------------------------------------------------------- +# Entry point. +# -------------------------------------------------------------------------------------------------- +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--baseline-exe-dir", required=True, + help="Directory of the baseline (released package) build.") + parser.add_argument("--current-exe-dir", required=True, + help="Directory of the candidate (from-source) build.") + parser.add_argument("--assembly", default="PerformanceTests.dll") + parser.add_argument("--results-dir", required=True) + parser.add_argument("--work-dir", default=None, + help="Scratch directory for per-unit run dirs (default: /../perf-interleave-work).") + parser.add_argument("--threshold", type=float, default=10.0) + parser.add_argument("--reps", type=int, default=3, + help="Total interleaved passes for a flagged unit (best-of-N). 1 disables confirmation.") + parser.add_argument("--baseline-version", default="baseline") + parser.add_argument("--baseline-runner-config", default=None, + help="Override RUNNER_CONFIG for baseline-variant subprocesses only " + "(e.g. to force a different SqlClient behaviour flag for the " + "baseline, such as comparing the legacy vs new connection pool " + "from the same build). Omit to use the ambient RUNNER_CONFIG for " + "both variants (default behaviour).") + parser.add_argument("--current-runner-config", default=None, + help="Override RUNNER_CONFIG for current-variant subprocesses only.") + parser.add_argument("--client-cpus", default=os.environ.get("PERF_CLIENT_CPUS", ""), + help="CPU set to pin the benchmark client to, e.g. '16-31'.") + parser.add_argument("--fail-on-regression", action="store_true", + help="Exit non-zero if any CONFIRMED regression is detected.") + args = parser.parse_args(argv) + + if args.reps < 1: + parser.error("--reps must be >= 1") + + results_dir = os.path.abspath(args.results_dir) + work_dir = os.path.abspath(args.work_dir) if args.work_dir \ + else os.path.join(os.path.dirname(results_dir), "perf-interleave-work") + os.makedirs(work_dir, exist_ok=True) + os.makedirs(results_dir, exist_ok=True) + + cpus = parse_cpus(args.client_cpus) + if cpus: + print(f"Pinning benchmark client to CPUs {args.client_cpus} ({len(cpus)} core(s)).") + else: + print("PERF_CLIENT_CPUS not set; running without CPU pinning.", file=sys.stderr) + + runner = Runner(os.path.abspath(args.baseline_exe_dir), + os.path.abspath(args.current_exe_dir), + args.assembly, work_dir, cpus, + baseline_runner_config=args.baseline_runner_config, + current_runner_config=args.current_runner_config) + + units = runner.list_units() + if not units: + print("ERROR: no enabled benchmark units were reported by the current build.", file=sys.stderr) + return 1 + print(f"Enabled units ({len(units)}): {', '.join(units)}") + + entries, confirmed, unconfirmed = orchestrate( + runner, units, results_dir, args.threshold, args.reps) + + # Outputs. + comparison_dir = os.path.join(results_dir, "comparison") + md = render_markdown(entries, confirmed, unconfirmed, args.baseline_version, args.threshold, args.reps) + with open(os.path.join(comparison_dir, "comparison.md"), "w", encoding="utf-8") as fh: + fh.write(md + "\n") + with open(os.path.join(comparison_dir, "comparison.json"), "w", encoding="utf-8") as fh: + json.dump({ + "baselineVersion": args.baseline_version, + "thresholdPct": args.threshold, + "confirmationRuns": args.reps, + "confirmedRegressions": len(confirmed), + "unconfirmedRegressions": len(unconfirmed), + "entries": entries, + }, fh, indent=2) + # Surface the comparison as the top-level run summary (collect-results attaches results/*.md). + shutil.copyfile(os.path.join(comparison_dir, "comparison.md"), + os.path.join(results_dir, "summary.md")) + + print(f"Interleaved comparison complete: {len(confirmed)} confirmed regression(s), " + f"{len(unconfirmed)} unconfirmed.") + + if args.fail_on_regression and confirmed: + print("Confirmed regressions detected; failing as requested.", file=sys.stderr) + for e in confirmed: + print(f" {e['key']}: {compare_perf._fmt_pct(e['meanDeltaPct'])} " + f"({e['regressionReps']}/{e['totalReps']} reps)", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/perf_to_kusto.py b/eng/pipelines/perf/scripts/perf_to_kusto.py new file mode 100644 index 0000000000..15a9790902 --- /dev/null +++ b/eng/pipelines/perf/scripts/perf_to_kusto.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +"""Translate BenchmarkDotNet "full" JSON reports into Kusto perf-results rows. + +Implements the mapping documented on the InternalDriverTools wiki, page 270 +("Performance Results Database Specification") and the SqlClient conversion +subpage. Produces two newline-delimited JSON (NDJSON) files ready for Kusto +ingestion: + + * PerfRun - one row describing this run (baseline OR current). + * PerfBenchmarkResult - one row per benchmark method/parameter combination. + +All benchmark timings in BenchmarkDotNet are expressed in NANOSECONDS; the +schema stores milliseconds, so every time value is divided by 1,000,000. + +Only the Python standard library is required. +""" + +import argparse +import datetime +import glob +import json +import os +import re +import sys + + +NS_PER_MS = 1_000_000.0 + + +def _utcnow_iso(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +# Top-level runner-config keys that must never be captured into PerfRun.Config: +# * ConnectionString - contains server/credential details (secret; not a comparable knob). +# * Benchmarks - per-runner tuning object, not a boolean feature flag. +_CONFIG_EXCLUDED_KEYS = frozenset(("ConnectionString", "Benchmarks")) + + +def _load_runner_config(path): + """Return the runner's boolean feature flags as a dict for the PerfRun.Config column. + + The benchmark runner is driven by a .jsonc config (see PerformanceTests/runnerconfig.jsonc). + Only its top-level *boolean* flags describe the SqlClient behaviour a run exercised + (e.g. UseManagedSniOnWindows, UseOptimizedAsyncBehaviour), so those are the values worth + recording alongside the results. ConnectionString and Benchmarks are excluded, and any + non-boolean top-level entry is ignored so the shape stays a flat {flag: bool} map. + + Returns an empty dict (and warns) when the file is missing or unparseable, so a config + problem degrades to an empty Config rather than failing the whole translation. + """ + if not path: + return {} + try: + with open(path, "r", encoding="utf-8-sig") as handle: + raw = handle.read() + except OSError as exc: + print(f"WARNING: could not read runner config {path}: {exc}", file=sys.stderr) + return {} + # Strip //-style line comments so the .jsonc content parses as plain JSON. The config uses + # only whole-line comments, so a leading-whitespace match is sufficient and never touches a + # '//' embedded in a string value. + no_comments = re.sub(r"(?m)^\s*//.*$", "", raw) + try: + cfg = json.loads(no_comments) + except ValueError as exc: + print(f"WARNING: could not parse runner config {path}: {exc}", file=sys.stderr) + return {} + if not isinstance(cfg, dict): + print(f"WARNING: runner config {path} is not a JSON object; ignoring.", file=sys.stderr) + return {} + return {key: value for key, value in cfg.items() + if key not in _CONFIG_EXCLUDED_KEYS and isinstance(value, bool)} + + +def _parse_config_overrides(pairs): + """Parse repeated ``--config-override NAME=VALUE`` items into a ``{flag: bool}`` dict. + + The perf pipeline can set a handful of SqlClient behaviour flags (e.g. UseConnectionPoolV2) + at queue time. Those values are applied to the runner config the benchmarks actually run + against, but the checked-in ``runnerconfig.jsonc`` this script reads still holds the defaults, + so the queue-time values are threaded in here to keep ``PerfRun.Config`` faithful to the run. + + Only boolean values are accepted (Config is a flat ``{flag: bool}`` map); a non-boolean value + is warned about and skipped so a typo never injects a non-bool. Later duplicates win.""" + overrides = {} + for item in pairs or []: + name, sep, value = item.partition("=") + name = name.strip() + token = value.strip().lower() + if not sep or not name: + print(f"WARNING: ignoring --config-override '{item}' (expected NAME=VALUE).", + file=sys.stderr) + continue + if token in ("true", "1", "yes"): + overrides[name] = True + elif token in ("false", "0", "no"): + overrides[name] = False + else: + print(f"WARNING: ignoring --config-override '{item}' " + f"(value '{value}' is not a boolean).", file=sys.stderr) + return overrides + + +def _normalize_os(value): + """Map an OS descriptor to the schema's canonical 'Windows' or 'Linux'.""" + if not value: + return "" + v = value.strip().lower() + if "win" in v: + return "Windows" + if any(tok in v for tok in ("linux", "unix", "ubuntu", "debian", "centos", "alpine")): + return "Linux" + return value + + +def _normalize_arch(value): + """Map a process-architecture descriptor to the schema's lowercase form (e.g. X64 -> x64).""" + if not value: + return "" + return value.strip().lower() + + +def _normalize_run_type(value): + """Map the harness run mode to the schema's 'Sequential' or 'Interweaved'.""" + if not value: + return "" + v = value.strip().lower() + if v.startswith("seq"): + return "Sequential" + if v.startswith("inter"): # interleaved (harness) -> Interweaved (schema spelling) + return "Interweaved" + return value + + +def _branch_category(branch_name): + """Map a git ref to one of the schema's BranchCategory buckets.""" + if not branch_name: + return "other" + name = branch_name + for prefix in ("refs/heads/", "refs/"): + if name.startswith(prefix): + name = name[len(prefix):] + break + if name.startswith("pull/") or branch_name.startswith("refs/pull/"): + return "pull_request" + # The internal ADO mirror prefixes its branches with 'internal/', so 'internal/main' and + # 'internal/release/*' are the same branches as their public counterparts and must land in the + # same buckets - otherwise mirrored runs would all be categorised as 'other'. + if name.startswith("internal/"): + name = name[len("internal/"):] + if name == "main" or name == "master": + return "main" + if name.startswith("release/"): + return "release" + if name.startswith("dev/"): + return "dev" + if name.startswith("feat/") or name.startswith("feature/"): + return "feature" + return "other" + + +def _parse_parameters(parameters): + """Parse BenchmarkDotNet's 'Parameters' string into a structured bag. + + Example input: "RowCount=1000, UseAsync=True" + """ + bag = {} + if not parameters: + return bag + for part in parameters.split(","): + part = part.strip() + if not part or "=" not in part: + continue + key, _, value = part.partition("=") + bag[key.strip()] = value.strip() + return bag + + +def _driver_specific_metrics(bench): + metrics = {} + memory = bench.get("Memory") or {} + for field in ("Gen0Collections", "Gen1Collections", "Gen2Collections", + "TotalOperations", "BytesAllocatedPerOperation"): + if field in memory and memory[field] is not None: + metrics[field] = memory[field] + for metric in bench.get("Metrics", []) or []: + descriptor = metric.get("Descriptor") or {} + legend = descriptor.get("Legend") or descriptor.get("Id") + if legend is not None and metric.get("Value") is not None: + metrics[legend] = metric["Value"] + return metrics + + +def _percentile(stats, name): + pct = stats.get("Percentiles") or {} + value = pct.get(name) + return (value / NS_PER_MS) if value is not None else None + + +def translate(input_dir, ctx): + """Return (run_row, [result_rows]) for every full-report JSON in input_dir.""" + derived_run_id = "{driver}|{commit}|{pipeline}".format( + driver=ctx["driver_name"], + commit=ctx["commit_hash"], + pipeline=ctx["pipeline_run_id"], + ) + now = _utcnow_iso() + + run_row = { + "DerivedRunId": derived_run_id, + "DriverName": ctx["driver_name"], + "MachineName": ctx["machine_name"], + "AgentName": ctx["agent_name"], + "OperatingSystem": _normalize_os(ctx.get("operating_system")), + "Architecture": _normalize_arch(ctx.get("architecture")), + "RunType": _normalize_run_type(ctx.get("run_type")), + "PipelineRunId": ctx["pipeline_run_id"], + "BuildUrl": ctx["build_url"], + "RunDate": ctx["run_date"] or now, + "BranchName": ctx["branch_name"], + "BranchCategory": _branch_category(ctx["branch_name"]), + "VersionString": ctx["version_string"], + "CommitHash": ctx["commit_hash"], + "CommitDate": ctx["commit_date"] or None, + "IsComparableBase": bool(ctx["is_comparable_base"]), + "Config": ctx.get("config") or {}, + "IngestedAt": now, + } + + result_rows = [] + runtime_seen = None + platform_seen = None + os_seen = None + + pattern = os.path.join(input_dir, "**", "*-report-full.json") + for path in sorted(glob.glob(pattern, recursive=True)): + try: + with open(path, "r", encoding="utf-8-sig") as handle: + data = json.load(handle) + except (OSError, ValueError) as exc: + print(f"WARNING: could not parse {path}: {exc}", file=sys.stderr) + continue + + host = data.get("HostEnvironmentInfo") or {} + runtime = host.get("RuntimeVersion") + architecture = host.get("Architecture") + runtime_seen = runtime_seen or runtime + platform_seen = platform_seen or architecture + os_seen = os_seen or host.get("OsVersion") + + for bench in data.get("Benchmarks", []): + stats = bench.get("Statistics") or {} + mean_ns = stats.get("Mean") + if mean_ns is None: + continue + mean_ms = mean_ns / NS_PER_MS + + btype = bench.get("Type", "") + method = bench.get("Method", "") + params = bench.get("Parameters", "") or "" + memory = bench.get("Memory") or {} + + benchmark_id = "{run}|{name}|{method}|{sig}".format( + run=derived_run_id, name=btype, method=method, sig=params) + + result_rows.append({ + "BenchmarkId": benchmark_id, + "DerivedRunId": derived_run_id, + "BenchmarkName": btype, + "MethodName": method, + "ParameterSignature": params, + "ParameterBag": _parse_parameters(params), + "JobName": (bench.get("Job") or bench.get("JobConfig") or ""), + "MeanMs": mean_ms, + "P50Ms": _percentile(stats, "P50"), + "P95Ms": _percentile(stats, "P95"), + "P99Ms": _percentile(stats, "P99"), + "ThroughputOpsPerSec": (1000.0 / mean_ms) if mean_ms else None, + "ErrorMs": (stats.get("StandardError") / NS_PER_MS) + if stats.get("StandardError") is not None else None, + "StdDevMs": (stats.get("StandardDeviation") / NS_PER_MS) + if stats.get("StandardDeviation") is not None else None, + "MemoryAllocatedBytes": memory.get("BytesAllocatedPerOperation"), + "Runtime": runtime, + "Platform": architecture, + "DriverSpecificMetrics": _driver_specific_metrics(bench), + "IngestedAt": now, + }) + + # Fill OperatingSystem / Architecture from the benchmark host environment when the pipeline did + # not supply explicit overrides, normalizing to the schema's canonical values. + if not run_row["OperatingSystem"]: + run_row["OperatingSystem"] = _normalize_os(os_seen) + if not run_row["Architecture"]: + run_row["Architecture"] = _normalize_arch(platform_seen) + + return run_row, result_rows + + +def _write_ndjson(path, rows): + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, separators=(",", ":")) + "\n") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", required=True, + help="Directory containing *-report-full.json for one pass.") + parser.add_argument("--out-run", required=True, help="PerfRun NDJSON output path.") + parser.add_argument("--out-results", required=True, + help="PerfBenchmarkResult NDJSON output path.") + parser.add_argument("--driver-name", default="Microsoft.Data.SqlClient") + parser.add_argument("--machine-name", default="") + parser.add_argument("--agent-name", default="") + parser.add_argument("--operating-system", default="", + help="OS the run executed on (Windows/Linux). " + "Derived from the benchmark host env when omitted.") + parser.add_argument("--architecture", default="", + help="Process architecture the run executed on (x64/x86). " + "Derived from the benchmark host env when omitted.") + parser.add_argument("--run-type", default="", + help="Benchmark execution ordering (Sequential/Interweaved).") + parser.add_argument("--pipeline-run-id", required=True) + parser.add_argument("--build-url", default="") + parser.add_argument("--run-date", default="") + parser.add_argument("--branch-name", default="") + parser.add_argument("--version-string", default="") + parser.add_argument("--commit-hash", required=True) + parser.add_argument("--commit-date", default="") + parser.add_argument("--is-comparable-base", default="false") + parser.add_argument("--runner-config", default="", + help="Path to the benchmark runner's .jsonc config. Its top-level boolean " + "flags (excluding ConnectionString and Benchmarks) are recorded in " + "PerfRun.Config.") + parser.add_argument("--config-override", action="append", default=[], + metavar="NAME=BOOL", + help="Override or add a PerfRun.Config boolean flag (repeatable). Values " + "supplied here win over those read from --runner-config, so the pipeline " + "records the flags it actually set (e.g. UseConnectionPoolV2=true) " + "regardless of the checked-in runnerconfig.jsonc defaults.") + args = parser.parse_args(argv) + + config = _load_runner_config(args.runner_config) + config.update(_parse_config_overrides(args.config_override)) + + ctx = { + "driver_name": args.driver_name, + "machine_name": args.machine_name, + "agent_name": args.agent_name, + "operating_system": args.operating_system, + "architecture": args.architecture, + "run_type": args.run_type, + "pipeline_run_id": args.pipeline_run_id, + "build_url": args.build_url, + "run_date": args.run_date, + "branch_name": args.branch_name, + "version_string": args.version_string, + "commit_hash": args.commit_hash, + "commit_date": args.commit_date, + "is_comparable_base": str(args.is_comparable_base).lower() in ("1", "true", "yes"), + "config": config, + } + + run_row, result_rows = translate(args.input_dir, ctx) + _write_ndjson(args.out_run, [run_row]) + _write_ndjson(args.out_results, result_rows) + + print(f"Wrote 1 PerfRun row -> {args.out_run}") + print(f"Wrote {len(result_rows)} PerfBenchmarkResult row(s) -> {args.out_results}") + if not result_rows: + print("WARNING: no benchmark results were translated.", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eng/pipelines/perf/scripts/run-perf-tests.ps1 b/eng/pipelines/perf/scripts/run-perf-tests.ps1 new file mode 100644 index 0000000000..2b3f9bed40 --- /dev/null +++ b/eng/pipelines/perf/scripts/run-perf-tests.ps1 @@ -0,0 +1,855 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### +# +# run-perf-tests.ps1 +# +# Entry point executed ON the Perf Test Lab Windows VM by the InternalDriverTools/PerfTest extends +# template (v1/Perf.Test.Job.yml). The template SCPs the driver source tree to the VM, runs this +# script over SSH, then SCPs the results sub-directory back and publishes it as a pipeline artifact. +# +# This is the Windows counterpart of run-perf-tests.sh. See that file for the full description of +# responsibilities. On Windows the benchmark client is pinned to the reserved CPU set via the +# process ProcessorAffinity mask (derived from PERF_CLIENT_CPUS) instead of taskset. +# +# Environment variables injected by the template (see wiki "Performance Test Automation"): +# SQL_SERVER Host/IP of the SQL Server on the perf VM (e.g. localhost). +# SQL_PASSWORD SQL Server 'sa' password. +# PERF_CLIENT_CPUS Core range reserved for the test client, e.g. "16-31". +# PERF_SQL_CPUS Core range SQL Server is pinned to, e.g. "0-15" (informational). +# +[CmdletBinding()] +param( + [string]$Configuration = "Release", + [string]$Framework = "net9.0", + [string]$ResultsSubdir = "perf-results", + [string]$BaselineVersion = "", + # Alternative to -BaselineVersion: benchmark against Microsoft.Data.SqlClient built from ANOTHER + # git ref of this repository (e.g. 'main') instead of a released NuGet package. Used by the PR + # perf pipeline, which compares the branch under test against the source it would merge into. + # The two baseline selectors are mutually exclusive. + [string]$BaselineSourceRef = "", + # Remote used to obtain the baseline ref when it cannot be fetched from the copied checkout's own + # 'origin' (e.g. the source tree reached the VM without its .git directory, or origin needs auth). + [string]$BaselineRepoUrl = "https://github.com/dotnet/SqlClient.git", + [string]$RegressionThreshold = "10", + # When set, a candidate-slower-than-baseline regression fails the run (wiki 339 §3 gate). + # Off by default so deltas are reported without blocking until the gate is trusted. + [switch]$FailOnRegression, + # Benchmark run model (wiki 339 §2.2/§2.3/§2.6): + # interleaved -> run one unit at a time, baseline and candidate back-to-back, with best-of-N + # confirmation of flagged regressions (the noise-resistant default). + # sequential -> legacy: run the whole baseline suite, then the whole candidate suite, compare. + [ValidateSet("interleaved", "sequential")] + [string]$RunMode = "interleaved", + # Best-of-N: total interleaved passes for a flagged unit before a regression is confirmed. + [ValidateRange(1, [int]::MaxValue)] + [int]$ConfirmationRuns = 3, + # Optional SqlClient behaviour flags (true/false, or empty to leave the checked-in + # runnerconfig.jsonc default untouched). Written into the runner config the benchmarks run + # against and, via the pipeline's Kusto translation, recorded in PerfRun.Config. + [ValidateSet("", "true", "false")] + [string]$UseManagedSniOnWindows = "", + [ValidateSet("", "true", "false")] + [string]$UseOptimizedAsyncBehaviour = "", + [ValidateSet("", "true", "false")] + [string]$UseConnectionPoolV2 = "", + # Alternative to -BaselineVersion/-BaselineSourceRef: an A/B experiment on ONE runner-config + # switch. Both passes build the SAME source; only the named switch differs (baseline=false, + # current=true), which is the only way to compare a switch whose value is latched process-wide + # (e.g. UseConnectionPoolV2 is read and cached the first time a pool is created). Mutually + # exclusive with the other two baseline selectors, and overrides the matching -Use* flag (which + # would otherwise be ambiguous: one value cannot describe two passes). ValidateSet restricts it + # to switches this script knows how to stamp, so a typo fails fast at binding time instead of + # silently writing an inert key and reporting a meaningless zero-delta comparison. + [ValidateSet("", "UseConnectionPoolV2", "UseOptimizedAsyncBehaviour", "UseManagedSniOnWindows")] + [string]$SwitchUnderTest = "" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +#################################################################################################### +# Native-command error handling +# +# The perf VM runs Windows PowerShell 5.1. There, with $ErrorActionPreference = 'Stop', ANY write +# to stderr by a native command (dotnet, python) is promoted to a TERMINATING error - even when the +# command's exit code is 0. The .NET CLI and Python routinely emit non-fatal diagnostics (SDK +# resolution notes, restore/build warnings, progress) to stderr, so an unguarded 'dotnet ...' or +# 'python3 ...' aborts the whole run and surfaces the tool's stderr text as the failure. +# +# PowerShell 7.3+ exposes $PSNativeCommandUseErrorActionPreference to opt out; set it where present. +# On 5.1 there is no such switch, so native tools are invoked through Invoke-Native, which relaxes +# the preference for the duration of the call and judges success solely by the process exit code. +#################################################################################################### + +if (Test-Path Variable:\PSNativeCommandUseErrorActionPreference) { + $PSNativeCommandUseErrorActionPreference = $false +} + +# Keep the checkout clean: never let the helper scripts drop __pycache__/*.pyc into eng/. +$env:PYTHONDONTWRITEBYTECODE = "1" + +# Run a native command (in a scriptblock) with stderr-as-terminating-error suppressed, then throw +# $FailureMessage if it exited non-zero. Use for native calls whose non-zero exit must fail the run. +function Invoke-Native { + [CmdletBinding()] + param( + [Parameter(Mandatory, Position = 0)][scriptblock] $Command, + [Parameter(Position = 1)][string] $FailureMessage = "Native command failed" + ) + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + & $Command + } finally { + $ErrorActionPreference = $previousPreference + } + if ($LASTEXITCODE -ne 0) { + throw "$FailureMessage (exit $LASTEXITCODE)." + } +} + +#################################################################################################### +# Resolve paths +#################################################################################################### + +# This script lives at /eng/pipelines/perf/scripts/run-perf-tests.ps1, so the repo root is +# four levels up. +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$RepoRoot = (Resolve-Path (Join-Path $ScriptDir "..\..\..\..")).Path +$PerfProject = Join-Path $RepoRoot "src\Microsoft.Data.SqlClient\tests\PerformanceTests\Microsoft.Data.SqlClient.PerformanceTests.csproj" +$PerfDir = Split-Path -Parent $PerfProject +$ResultsDir = Join-Path $RepoRoot $ResultsSubdir + +$SqlServer = if ($env:SQL_SERVER) { $env:SQL_SERVER } else { "localhost" } +$SqlPassword = $env:SQL_PASSWORD +$DbName = "sqlclient-perf-db" + +Write-Host "==================================================================" +Write-Host " SqlClient Performance Tests" +Write-Host "==================================================================" +Write-Host " Repo root : $RepoRoot" +Write-Host " Perf project : $PerfProject" +Write-Host " Configuration : $Configuration" +Write-Host " Framework : $Framework" +Write-Host " Results dir : $ResultsDir" +Write-Host " Run mode : $RunMode (confirmation runs: $ConfirmationRuns)" +Write-Host " Baseline ver : $(if ($BaselineVersion) { $BaselineVersion } else { '' })" +Write-Host " Baseline ref : $(if ($BaselineSourceRef) { $BaselineSourceRef } else { '' })" +Write-Host " Switch A/B : $(if ($SwitchUnderTest) { "$SwitchUnderTest (baseline=false vs current=true)" } else { '' })" +Write-Host " SQL_SERVER : $SqlServer" +Write-Host " PERF_CLIENT_CPUS: $($env:PERF_CLIENT_CPUS)" +Write-Host " PERF_SQL_CPUS : $($env:PERF_SQL_CPUS)" +Write-Host "==================================================================" + +if (-not (Test-Path $PerfProject)) { + throw "Performance test project not found at $PerfProject" +} +# The two baseline selectors describe different builds of the same "baseline" pass, so requesting +# both is always a mistake; fail fast rather than silently honouring one of them. +if ((-not [string]::IsNullOrEmpty($BaselineVersion)) -and (-not [string]::IsNullOrEmpty($BaselineSourceRef))) { + throw "-BaselineVersion and -BaselineSourceRef are mutually exclusive." +} +if ((-not [string]::IsNullOrEmpty($SwitchUnderTest)) -and ((-not [string]::IsNullOrEmpty($BaselineVersion)) -or (-not [string]::IsNullOrEmpty($BaselineSourceRef)))) { + throw "-SwitchUnderTest is mutually exclusive with -BaselineVersion and -BaselineSourceRef: it compares the SAME source build with one switch flipped, so mixing in a source change would make the delta unattributable." +} +# UseManagedSniOnWindows only selects an implementation on Windows. PowerShell Core also runs on +# Linux, so check the host rather than assume this script implies Windows: off-Windows both passes +# use managed SNI and the experiment reports a ~0% delta that reads as "the switch is free". +# $IsWindows is undefined on Windows PowerShell 5.1, which is itself Windows-only, so null is Windows. +if ($SwitchUnderTest -eq "UseManagedSniOnWindows") { + $onWindows = if ($null -eq (Get-Variable -Name IsWindows -ErrorAction SilentlyContinue)) { $true } else { $IsWindows } + if (-not $onWindows) { + throw "-SwitchUnderTest UseManagedSniOnWindows is Windows-only; elsewhere managed SNI is always used, so baseline and current would be identical. Re-run this experiment on the Windows platform." + } +} +# -SwitchUnderTest forces its switch explicitly for each pass (baseline=false, current=true), so a +# separately-supplied -Use* flag for that SAME switch would be silently overridden; warn rather than +# let that go unnoticed. Other -Use* flags still apply normally to both passes. +$conflictingFlag = "" +$conflictingFlagValue = switch ($SwitchUnderTest) { + "UseConnectionPoolV2" { $conflictingFlag = "-UseConnectionPoolV2"; $UseConnectionPoolV2 } + "UseOptimizedAsyncBehaviour" { $conflictingFlag = "-UseOptimizedAsyncBehaviour"; $UseOptimizedAsyncBehaviour } + "UseManagedSniOnWindows" { $conflictingFlag = "-UseManagedSniOnWindows"; $UseManagedSniOnWindows } + default { "" } +} +if (-not [string]::IsNullOrEmpty($conflictingFlagValue)) { + Write-Warning "$conflictingFlag=$conflictingFlagValue is ignored when -SwitchUnderTest is $SwitchUnderTest (baseline forces false, current forces true)." +} +if ([string]::IsNullOrEmpty($SqlPassword)) { + throw "SQL_PASSWORD environment variable is not set (expected from the perf template)." +} + +New-Item -ItemType Directory -Force -Path $ResultsDir | Out-Null + +# Record VM-side run metadata (e.g. the perf VM hostname) for the agent-side Kusto translation. +"MACHINE_NAME=$env:COMPUTERNAME" | Set-Content -Path (Join-Path $ResultsDir "runinfo.env") -Encoding ASCII + +$env:DOTNET_NOLOGO = "1" +$env:DOTNET_CLI_TELEMETRY_OPTOUT = "1" +$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = "1" + +#################################################################################################### +# 1. Install the .NET SDK (pinned by global.json) and the runtimes for the target frameworks. +#################################################################################################### + +function Install-DotNet { + $globalJson = Get-Content (Join-Path $RepoRoot "global.json") -Raw + # Strip // comments so ConvertFrom-Json accepts the file. + $globalJson = ($globalJson -split "`n" | ForEach-Object { $_ -replace '//.*$', '' }) -join "`n" + $sdkVersion = (ConvertFrom-Json $globalJson).sdk.version + if ([string]::IsNullOrEmpty($sdkVersion)) { + throw "Could not determine SDK version from global.json" + } + + $dotnetRoot = Join-Path $env:USERPROFILE ".dotnet" + $env:DOTNET_ROOT = $dotnetRoot + $env:PATH = "$dotnetRoot;$dotnetRoot\tools;$env:PATH" + + Write-Host "Installing .NET SDK $sdkVersion into $dotnetRoot ..." + $installScript = Join-Path $env:TEMP "dotnet-install.ps1" + Invoke-WebRequest -UseBasicParsing "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript + + & $installScript -Version $sdkVersion -InstallDir $dotnetRoot + foreach ($channel in @("8.0", "9.0", "10.0")) { + & $installScript -Channel $channel -Runtime dotnet -InstallDir $dotnetRoot + } +} + +$hasNet10Sdk = $false +if (Get-Command dotnet -ErrorAction SilentlyContinue) { + # 'dotnet --version' evaluated from the repo root honours global.json (including rollForward), so + # it succeeds only when the pinned SDK is actually installed. A bare '10.0.*' match would accept + # the wrong SDK band and skip installing the pinned one. + Push-Location $RepoRoot + try { + # Relax Stop here: a missing/mismatched pinned SDK makes 'dotnet --version' write to stderr + # and exit non-zero, which under Stop would abort before we can fall through to Install-DotNet. + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + dotnet --version *> $null + if ($LASTEXITCODE -eq 0) { $hasNet10Sdk = $true } + } finally { + $ErrorActionPreference = $previousPreference + } + } finally { + Pop-Location + } +} +if ($hasNet10Sdk) { + Write-Host "Using pre-installed dotnet: $((Get-Command dotnet).Source)" +} else { + Install-DotNet +} + +# Informational only: never let 'dotnet --info' (or its stderr) fail the run. +try { Invoke-Native { dotnet --info } "dotnet --info failed" } catch { Write-Warning $_.Exception.Message } + +#################################################################################################### +# 2. Create the perf database on the VM's SQL Server. +# +# The benchmark runners create their own tables but not the database, so create it here +# (idempotently) using sqlcmd. sqlcmd is required on the VM; if it is not present the script fails +# fast (throws below) rather than continuing on to run benchmarks against a missing database. +#################################################################################################### + +Write-Host "Ensuring database [$DbName] exists on $SqlServer ..." + +$sqlcmd = Get-Command sqlcmd -ErrorAction SilentlyContinue +if ($sqlcmd) { + # Relax Stop around the native sqlcmd call so a benign stderr write cannot abort the run before + # the explicit exit-code check below (Windows PowerShell 5.1 promotes native stderr under Stop). + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 30 ` + -Q "IF DB_ID('$DbName') IS NULL CREATE DATABASE [$DbName];" + } finally { + $ErrorActionPreference = $previousPreference + } + if ($LASTEXITCODE -ne 0) { throw "sqlcmd failed to create database [$DbName] (exit $LASTEXITCODE)." } + Write-Host "Database [$DbName] is ready." +} else { + throw "sqlcmd was not found on the VM; cannot create the perf database [$DbName]." +} + +#################################################################################################### +# Noise-reduction controls (InternalDriverTools wiki 339, "Reducing Noise in Performance Tests"). +# +# The Perf Test Lab already provides the isolated dedicated host, the tuned SQL instance and the +# disjoint client CPU set (PERF_CLIENT_CPUS, pinned per pass below). These are the remaining +# harness-owned controls: per-run diagnostics, a fail-loud preflight and a warm-up, so a run's +# mean/variance is steadier and a broken run cannot masquerade as a pass. (The glibc allocator and +# sysctl tuning from the Linux harness are Linux-only and intentionally omitted here.) +#################################################################################################### + +$DiagDir = Join-Path $ResultsDir "diagnostics" +New-Item -ItemType Directory -Force -Path $DiagDir | Out-Null + +# --- §2.11 Capture host CPU topology (static, once per run) --------------------------------------- +try { + Get-CimInstance Win32_Processor | + Select-Object Name, NumberOfCores, NumberOfLogicalProcessors, MaxClockSpeed, CurrentClockSpeed | + Format-List | Out-File -FilePath (Join-Path $DiagDir "cpu-info.txt") -Encoding UTF8 +} catch { Write-Warning "Could not capture CPU info: $_" } + +# --- §2.11 Capture the SQL instance configuration (confirm the lab tuning actually took effect) --- +try { + & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 30 -h -1 -W ` + -Q "SET NOCOUNT ON; + SELECT name, value_in_use FROM sys.configurations + WHERE name IN ('max degree of parallelism','cost threshold for parallelism', + 'max server memory (MB)','min server memory (MB)','affinity mask', + 'affinity I/O mask'); + SELECT 'tempdb_data_files' AS setting, COUNT(*) AS value FROM tempdb.sys.database_files WHERE type = 0; + SELECT @@VERSION;" ` + *> (Join-Path $DiagDir "sql-config.txt") + Write-Host "Captured SQL instance config -> $(Join-Path $DiagDir 'sql-config.txt')" +} catch { Write-Warning "Could not capture SQL instance config: $_" } + +# --- §2.10 / §2.5 Fail loud on an unreachable server, and warm the buffer pool / plan cache ------- +# A benchmark suite that "skips" when the server is down produces an empty comparison that reads +# green; verify connectivity up front and touch the target DB before the first measured benchmark. +$previousPreference = $ErrorActionPreference +$ErrorActionPreference = 'Continue' +try { + & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 15 ` + -Q "SET NOCOUNT ON; USE [$DbName]; SELECT 1;" *> $null +} finally { + $ErrorActionPreference = $previousPreference +} +if ($LASTEXITCODE -ne 0) { + throw "SQL Server $SqlServer (db $DbName) is unreachable; refusing to run so an empty perf comparison cannot be reported as a pass." +} +Write-Host "Preflight: SQL Server $SqlServer (db $DbName) is reachable and warmed." + +#################################################################################################### +# 3. Inject the VM's SQL Server connection string into the benchmark runner config. +#################################################################################################### + +$RunnerConfig = Join-Path $RepoRoot "perf-runnerconfig.json" +$env:RUNNER_CONFIG = $RunnerConfig + +# The perf app also loads datatypes.json via the DATATYPES_CONFIG env var, falling back to +# "datatypes.json" in the working directory. Each pass runs from an otherwise-empty +# perf-run-