Skip to content

Commit b66f0b0

Browse files
authored
feat(bi): power bi native connector (#4563)
* step 1 * step 2 * step 3 * bots feedbacks * doc for release * switch build sdk * fixup * product docs * fixup
1 parent 8400ef4 commit b66f0b0

26 files changed

Lines changed: 2658 additions & 90 deletions
Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
name: Power BI connector
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "automation/powerbi/**"
7+
- ".github/workflows/powerbi-connector.yml"
8+
push:
9+
branches: [main]
10+
paths:
11+
- "automation/powerbi/**"
12+
tags:
13+
- "powerbi-v*"
14+
workflow_dispatch:
15+
16+
permissions:
17+
contents: read
18+
19+
jobs:
20+
build:
21+
runs-on: windows-latest
22+
steps:
23+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
24+
with:
25+
persist-credentials: false
26+
27+
- name: Check tag matches connector version
28+
if: startsWith(github.ref, 'refs/tags/powerbi-v')
29+
shell: pwsh
30+
run: |
31+
$pq = Get-Content automation/powerbi/connector/CisoAssistant.pq -Raw
32+
if ($pq -notmatch '\[Version = "(\d+\.\d+\.\d+)"\]') { throw "no [Version] attribute in CisoAssistant.pq" }
33+
$version = $Matches[1]
34+
$tag = $env:GITHUB_REF_NAME -replace '^powerbi-v', ''
35+
if ($version -ne $tag) { throw "tag $env:GITHUB_REF_NAME does not match connector [Version = `"$version`"]" }
36+
37+
- name: Build .mez (zip of connector sources)
38+
shell: pwsh
39+
run: |
40+
$connector = "automation/powerbi/connector"
41+
$staging = "obj/mez"
42+
New-Item -ItemType Directory -Force -Path $staging, "dist" | Out-Null
43+
Copy-Item "$connector/CisoAssistant.pq", "$connector/resources.resx", "$connector/*.png" -Destination $staging
44+
Compress-Archive -Path "$staging/*" -DestinationPath "dist/CisoAssistant.zip" -Force
45+
Move-Item "dist/CisoAssistant.zip" "dist/CisoAssistant.mez"
46+
47+
- name: Install Power Query SDK tools
48+
shell: pwsh
49+
run: |
50+
# Version-pinned: this tool is handed the signing certificate, so
51+
# "latest" is a supply-chain risk. Bump deliberately (and re-check
52+
# the binding-redirect patch below still applies).
53+
nuget install Microsoft.PowerQuery.SdkTools -Version 2.155.2 -OutputDirectory sdktools -NonInteractive
54+
$makepqx = Get-ChildItem -Path sdktools -Recurse -Filter MakePQX.exe | Select-Object -First 1
55+
if (-not $makepqx) { throw "MakePQX.exe not found" }
56+
Add-Content $env:GITHUB_ENV "MAKEPQX=$($makepqx.FullName)"
57+
58+
# SdkTools 2.155.2 ships MakePQX.exe.config without binding redirects
59+
# for the System.Text.Json dependency chain, so `MakePQX verify`
60+
# crashes (FileLoadException on System.Threading.Tasks.Extensions)
61+
# while printing its report. Generate the redirects from the DLLs
62+
# actually present in the tools folder.
63+
- name: Patch MakePQX binding redirects
64+
shell: pwsh
65+
run: |
66+
$toolsDir = Split-Path $env:MAKEPQX
67+
$configPath = "$env:MAKEPQX.config"
68+
[xml]$cfg = if (Test-Path $configPath) { Get-Content $configPath -Raw } else { "<configuration />" }
69+
$ns = "urn:schemas-microsoft-com:asm.v1"
70+
$nsm = New-Object System.Xml.XmlNamespaceManager($cfg.NameTable)
71+
$nsm.AddNamespace("a", $ns)
72+
$runtime = $cfg.SelectSingleNode("/configuration/runtime")
73+
if (-not $runtime) { $runtime = $cfg.DocumentElement.AppendChild($cfg.CreateElement("runtime")) }
74+
$ab = $runtime.SelectSingleNode("a:assemblyBinding", $nsm)
75+
if (-not $ab) { $ab = $runtime.AppendChild($cfg.CreateElement("assemblyBinding", $ns)) }
76+
$names = @(
77+
"System.Threading.Tasks.Extensions", "System.Runtime.CompilerServices.Unsafe",
78+
"System.Memory", "System.Buffers", "System.Numerics.Vectors",
79+
"System.Text.Json", "System.Text.Encodings.Web", "Microsoft.Bcl.AsyncInterfaces"
80+
)
81+
foreach ($name in $names) {
82+
$dll = Join-Path $toolsDir "$name.dll"
83+
if (-not (Test-Path $dll)) { continue }
84+
$an = [Reflection.AssemblyName]::GetAssemblyName($dll)
85+
$token = -join ($an.GetPublicKeyToken() | ForEach-Object { $_.ToString("x2") })
86+
foreach ($old in @($ab.SelectNodes("a:dependentAssembly[a:assemblyIdentity/@name='$name']", $nsm))) {
87+
$ab.RemoveChild($old) | Out-Null
88+
}
89+
$da = $ab.AppendChild($cfg.CreateElement("dependentAssembly", $ns))
90+
$ai = $da.AppendChild($cfg.CreateElement("assemblyIdentity", $ns))
91+
$ai.SetAttribute("name", $name)
92+
$ai.SetAttribute("publicKeyToken", $token)
93+
$ai.SetAttribute("culture", "neutral")
94+
$br = $da.AppendChild($cfg.CreateElement("bindingRedirect", $ns))
95+
$br.SetAttribute("oldVersion", "0.0.0.0-$($an.Version)")
96+
$br.SetAttribute("newVersion", $an.Version.ToString())
97+
Write-Host "redirect: $name -> $($an.Version) ($token)"
98+
}
99+
$cfg.Save($configPath)
100+
101+
- name: Pack .pqx
102+
shell: pwsh
103+
run: |
104+
& $env:MAKEPQX pack -mz "dist/CisoAssistant.mez" -t "dist/CisoAssistant.pqx"
105+
if ($LASTEXITCODE -ne 0) { throw "MakePQX pack failed ($LASTEXITCODE)" }
106+
if (-not (Test-Path "dist/CisoAssistant.pqx")) { throw "pack produced no .pqx" }
107+
108+
- name: Check signing secrets
109+
id: signing
110+
shell: pwsh
111+
env:
112+
PBI_SIGNING_CERT: ${{ secrets.PBI_SIGNING_CERT }}
113+
run: |
114+
if ([string]::IsNullOrEmpty($env:PBI_SIGNING_CERT)) {
115+
Add-Content $env:GITHUB_OUTPUT "enabled=false"
116+
} else {
117+
Add-Content $env:GITHUB_OUTPUT "enabled=true"
118+
}
119+
120+
- name: Sign and verify .pqx
121+
if: steps.signing.outputs.enabled == 'true' && github.event_name != 'pull_request'
122+
shell: pwsh
123+
env:
124+
PBI_SIGNING_CERT: ${{ secrets.PBI_SIGNING_CERT }}
125+
PBI_SIGNING_PASSWORD: ${{ secrets.PBI_SIGNING_PASSWORD }}
126+
run: |
127+
$pfx = Join-Path $env:RUNNER_TEMP "signing.pfx"
128+
[IO.File]::WriteAllBytes($pfx, [Convert]::FromBase64String($env:PBI_SIGNING_CERT))
129+
& $env:MAKEPQX sign "dist/CisoAssistant.pqx" --certificate $pfx --password $env:PBI_SIGNING_PASSWORD --replace
130+
$signExit = $LASTEXITCODE
131+
Remove-Item $pfx
132+
if ($signExit -ne 0) { throw "MakePQX sign failed ($signExit)" }
133+
& $env:MAKEPQX verify "dist/CisoAssistant.pqx"
134+
if ($LASTEXITCODE -ne 0) { throw "MakePQX verify failed ($LASTEXITCODE)" }
135+
136+
- name: Check artifacts exist
137+
shell: pwsh
138+
run: |
139+
foreach ($f in "dist/CisoAssistant.mez", "dist/CisoAssistant.pqx") {
140+
if (-not (Test-Path $f) -or (Get-Item $f).Length -eq 0) { throw "missing or empty artifact: $f" }
141+
}
142+
143+
- name: Upload artifacts
144+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
145+
with:
146+
name: powerbi-connector
147+
path: |
148+
dist/CisoAssistant.mez
149+
dist/CisoAssistant.pqx
150+
151+
release:
152+
if: startsWith(github.ref, 'refs/tags/powerbi-v')
153+
needs: build
154+
runs-on: ubuntu-latest
155+
permissions:
156+
contents: write
157+
steps:
158+
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
159+
with:
160+
name: powerbi-connector
161+
path: dist
162+
- name: Publish release assets
163+
env:
164+
GH_TOKEN: ${{ github.token }}
165+
GH_REPO: ${{ github.repository }}
166+
run: |
167+
tag="${GITHUB_REF_NAME}"
168+
gh release view "$tag" >/dev/null 2>&1 || gh release create "$tag" --title "Power BI connector $tag" --generate-notes
169+
gh release upload "$tag" dist/CisoAssistant.mez dist/CisoAssistant.pqx --clobber

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,9 @@ backend.ps1
3434
backend/.win_native_dev_runserver.py
3535
# > Tools
3636
tools/convert_library_v2.bat
37+
38+
# > Power BI (working files embed instance data; only the .pbit template ships)
39+
automation/powerbi/**/*.pbix
40+
automation/powerbi/connector/bin/
41+
automation/powerbi/connector/obj/
42+
*.pfx

‎automation/powerbi/CONTRIBUTING.md‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# Developing the Power BI connector
2+
3+
## Layout
4+
5+
```text
6+
automation/powerbi/
7+
├── DESIGN.md # architecture decisions — read first
8+
├── README.md # user-facing install/usage doc
9+
├── contract.json # data contract shared by the connector and backend tests
10+
├── connector/ # Power Query SDK project (CisoAssistant.pq + assets)
11+
├── signing/ # certificate generation + customer trust runbooks
12+
└── samples/ # starter .pbit template
13+
```
14+
15+
The backend contract test lives at
16+
`backend/app_tests/api/test_api_powerbi_contract.py` and runs with the
17+
regular backend test suite: it asserts that every JSON path consumed by
18+
`connector/CisoAssistant.pq` (as declared in `contract.json`) is present in
19+
the live serializer output. **When you change columns, update
20+
`CisoAssistant.pq` and `contract.json` together.**
21+
22+
## Environment: Windows required for build/test
23+
24+
The M source (`CisoAssistant.pq`) can be edited anywhere (VS Code +
25+
`PowerQuery.vscode-powerquery` for syntax/IntelliSense on macOS/Linux), but
26+
building and evaluating requires Windows:
27+
28+
- The **Power Query SDK** VS Code extension (`PowerQuery.vscode-powerquery-sdk`)
29+
is Windows-only (wraps `PQTest.exe` / `MakePQX.exe`; not available for
30+
Mac Silicon, Rosetta doesn't work).
31+
- **Power BI Desktop** is Windows-only.
32+
33+
On Apple Silicon: Parallels Desktop + Windows 11 ARM (~5 GB RAM is enough),
34+
repo mounted via Parallels shared folders.
35+
36+
## Build & test loop (Windows VM)
37+
38+
1. Open `automation/powerbi/connector/` as the VS Code workspace.
39+
2. Install the **Power Query SDK** extension; run its *Setup workspace*
40+
task if prompted (regenerates `.vscode/settings.json`).
41+
3. **Set credential** task → auth kind *Key* → paste a PAT from your dev
42+
instance. Credentials bind to the exact base URL in the query file —
43+
change the URL and the SDK may prompt again (*Clear ALL credentials*
44+
resets a confused state).
45+
4. **Point the test query at your dev instance**: the URL is the literal on
46+
the last line of `CisoAssistant.query.pq`. This is a local, uncommitted
47+
edit — the repo keeps the neutral `https://localhost:8443` placeholder;
48+
never commit your instance URL. From a VM, `localhost` is the VM itself:
49+
target the host instead (Parallels: the Mac is `10.211.55.2`; run
50+
`ALLOWED_HOSTS=localhost,127.0.0.1,10.211.55.2 python manage.py runserver 0.0.0.0:8000`
51+
on the host and use `http://10.211.55.2:8000`). Plain `http://` is
52+
accepted for loopback/private hosts only; the connector rejects it for
53+
public hosts (PAT travels in a header).
54+
5. *Evaluate current file* task → results appear in the panel. To inspect a
55+
specific table or debug a filter, replace the last line with a drill
56+
expression, e.g.
57+
`CisoAssistant.Contents("...") {[Name = "Facts"]}[Data] {[Name = "Applied Controls"]}[Data]`
58+
(optionally wrapped in `Table.SelectRows(..., each [updated_at] >= #datetime(...))`
59+
to watch fold behaviour in the backend logs) — same rule: revert before
60+
committing.
61+
6. *Build* task (pick **MakePQX**, not MSBuild) → `bin/AnyCPU/Debug/connector.mez`.
62+
MakePQX names the output after the workspace *folder*; the MSBuild path
63+
(`CisoAssistant.proj`) names it `CisoAssistant.mez` after the project.
64+
`.vscode/settings.json` points at the MakePQX name.
65+
7. Copy the `.mez` to the Custom Connectors folder — resolve it via the
66+
known folder (OneDrive may redirect Documents, especially on fresh VMs):
67+
```powershell
68+
$dir = Join-Path ([Environment]::GetFolderPath("MyDocuments")) "Power BI Desktop\Custom Connectors"
69+
New-Item -ItemType Directory -Force $dir | Out-Null
70+
Copy-Item bin\AnyCPU\Debug\connector.mez "$dir\CisoAssistant.mez" -Force
71+
```
72+
Allow uncertified extensions in Power BI Desktop's security options,
73+
restart Desktop, and test **Get Data → CISO Assistant** end-to-end.
74+
8. Run *TestConnection* task before touching gateway-related code.
75+
76+
### Testing the signed .pqx locally
77+
78+
Pack + sign with the extension's bundled MakePQX (path under
79+
`%USERPROFILE%\.vscode\extensions\...powerquery-sdk...\.nuget\...\tools\`),
80+
then validate through Power BI Desktop, not MakePQX:
81+
82+
1. `MakePQX pack -mz bin\AnyCPU\Debug\connector.mez -t bin\AnyCPU\Debug\CisoAssistant.pqx`
83+
2. `MakePQX sign <pqx> --certificate <pfx> --password <pwd> --replace`
84+
3. Put the `.pqx` in the Custom Connectors folder and **remove the `.mez`**
85+
there (both present = duplicate connector).
86+
4. Add the signing thumbprint to
87+
`HKLM:\Software\Policies\Microsoft\Power BI Desktop\TrustedCertificateThumbprints`
88+
(see `../signing/trust-thumbprint.md`), set Desktop's Data Extensions
89+
security to **(Recommended)**, restart. Desktop refusing to load the
90+
connector at Recommended = signature or trust problem; loading = the
91+
full customer path works.
92+
93+
Known quirk (SdkTools 2.155.2, any host): `MakePQX verify` crashes with a
94+
`System.Threading.Tasks.Extensions` assembly-binding error while printing
95+
its report — the package's `MakePQX.exe.config` lacks binding redirects
96+
for the System.Text.Json dependency chain. CI patches the redirects into
97+
the config before running the tools (see the "Patch MakePQX binding
98+
redirects" step in the workflow); locally, either apply the same patch or
99+
skip `verify` and use the Desktop trust test above, which is the
100+
authoritative check anyway.
101+
102+
`.mez` is just a zip of the connector folder contents — CI
103+
(`.github/workflows/powerbi-connector.yml`) builds it with
104+
`Compress-Archive`, packs it to `.pqx` with `MakePQX`, and signs it when
105+
the signing secrets are configured (see `signing/generate-cert.md`).
106+
107+
## Local backend for testing
108+
109+
Any running CISO Assistant works. For a filter/pagination sanity check
110+
without Power BI:
111+
112+
```bash
113+
curl -H "Authorization: Token $PAT" \
114+
"https://localhost:8443/api/applied-controls/?limit=2&offset=0&updated_at__gte=2026-01-01T00:00:00Z" -k
115+
```
116+
117+
## Versioning & release
118+
119+
The connector version is the `[Version = "x.y.z"]` attribute on the first
120+
line of `CisoAssistant.pq` — semver, aligned with the compatibility policy:
121+
122+
- **patch** — fixes, no contract change
123+
- **minor** — additive only (new tables/columns/bridges); never breaks an
124+
existing report
125+
- **major** — renames/removals in the contract, auth or data source kind
126+
changes; requires migration notes and should be avoided (`contract.json`
127+
+ the backend contract test exist to keep changes additive)
128+
129+
Release: bump `[Version]` (and `contract.json`/docs if the surface changed),
130+
merge, then tag `powerbi-v<x.y.z>`. CI refuses tags that don't match the
131+
`.pq` version, then builds, packs, signs (if the signing secrets are
132+
present), uploads artifacts, and attaches `CisoAssistant.mez` +
133+
`CisoAssistant.pqx` to the GitHub Release. Customer upgrade = replace the
134+
file, restart Power BI (reports and credentials bind to the data source
135+
kind, not the file or version).

0 commit comments

Comments
 (0)