Hi
Below is the script:
$subscriptionID = "88888888888888888888888"
Select-AzureRmSubscription -SubscriptionId $subscriptionID
$resourceGroupName = "dp-datalake-vmie-dev-rg"
$environment = "dev"
$clusterName = "vmiea0-h8-hdi"
$sqlServerName = "dbsqlserverdatalakevmiedev"
$virtualNetworkId = "/subscriptions/e6840abb-6346-4c09-996e-1b84bb7436d4/resourceGroups/dp-datalake-vmie-dev-rg/providers/Microsoft.Network/virtualNetworks/vnet-vmie-dev"
$KeyVaultName = "kv-dp-hdi-prd"
### Cluster Scripts Variables ###
$location = "West Europe"
$clusterType = "Spark"
$clusterOS = "Linux"
$clusterNodes = 4
$workerNodeSize = "Standard_D3_v2"
$headNodeSize = "Standard_D12_v2"
$db_all = Get-AzureRmSqlDatabase -ServerName $sqlServerName -ResourceGroupName $ResourceGroupName
echo "server name"
echo $sqlServerName
echo $db_all
$oozieDb = $db_all.DatabaseName -match "oozie"
$oozieSqlServer = $sqlServerName
$hiveDb = $db_all.DatabaseName -match "hive"
echo $hiveDb[0]
$dbServer = Get-AzureRmSqlServer -ServerName $sqlServerName -ResourceGroupName $ResourceGroupName
$dbUserName = $dbServer.SqlAdministratorLogin
$db_pwd_secret = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name "${sqlServerName}-db-pwd"
$password = ConvertTo-SecureString $db_pwd_secret.secretValueText -AsPlainText -Force
$psCred = New-Object System.Management.Automation.PSCredential -ArgumentList ($dbUserName, $password)
$oozieCreds = $psCred
$hiveCreds = $psCred
echo $hiveCreds
$hiveSqlServer = $sqlServerName
$subscriptionID = "e6840abb-63468888888888888888d4"
$subnet = "${virtualNetworkId}/subnets/subnet-vmie-dev01"
Select-AzureRmSubscription -SubscriptionId $subscriptionID
$tenantID = (Get-AzContext).Tenant.TenantId
#Get Service principal Information
$servicePrincipal = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name "${clusterName}-AppId"
$objectId = $servicePrincipal.secretValueText
$serviceCertificate = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name "${clusterName}-Cert"
$certificateFilePath = $serviceCertificate.secretValue
$servicePwdCert = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name "${clusterName}-CertPwd"
$password = $servicePwdCert.secretValue
$storageRootPath = "/clusters/vmiea0-h8-hdi"
# PSCredentials
$Cluster_Short_Name = $clusterName.Substring(0,6)
$clusterLoginUserName = $Cluster_Short_Name+"-admin"
$clusterLoginPassword = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name "${clusterName}-Pwd"
$clusterLoginPasswordClear = ConvertTo-SecureString $clusterLoginPassword.secretValueText -AsPlainText -Force
$httpCredentials = New-Object System.Management.Automation.PSCredential -ArgumentList ($clusterLoginUserName, $clusterLoginPasswordClear)
$sshUserName = "ssh-"+$Cluster_Short_Name
$sshPassword = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name "${clusterName}-SShPwd"
$sshPasswordClear = ConvertTo-SecureString $sshPassword.secretValueText -AsPlainText -Force
$clusterCredential = New-Object System.Management.Automation.PSCredential -ArgumentList ($sshUserName, $sshPasswordClear)
$sshPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQ0W+8yVJlRmzmjEj4LGdhv44Z0EUIajjyvxlgWRyRgjzMn1qM9+C9ObwT8ga25TMUyuecbRCLhZ5jcsFMBbquxhqC+TcXm0kmwBIAUCb6LmE1s91h7XXdqKbUfNt6I3K2oDyrdOIaKyPa71FoE9ZKA46cHQlk+Qe0kCUS7kAt+Q/oFhplzhPvR2I7rUDoNjicOa7qhcMXRrx8sp6v4ElgErb8+ghNOIweUVk2YaTJq1d6vTb4Av8luzxS8TIrYG5m2HsrrxGuplT72DecWwleA8atjCEkAoUSZQVdzbAW4uEXkHTHfiJMj/dR8E5PW12EaaWaBNEUaj1
1"
New-AzureRmHDInsightClusterConfig `
-ClusterType $clusterType `
-HeadNodeSize $headNodeSize `
-WorkerNodeSize $workerNodeSize `
| Add-AzureRmHDInsightMetastore `
-SqlAzureServerName "$oozieSqlServer.database.windows.net" `
-DatabaseName $oozieDb[0] `
-Credential $oozieCreds `
-MetastoreType OozieMetastore `
| Add-AzureRmHDInsightMetastore `
-SqlAzureServerName "$hiveSqlServer.database.windows.net" `
-DatabaseName $hiveDb[0] `
-Credential $hiveCreds `
-MetastoreType HiveMetastore `
| Add-AzureRmHDInsightComponentVersion `
-ComponentName "Spark" `
-ComponentVersion "2.6" `
| New-AzureRmHDInsightCluster `
-ClusterType $clusterType `
-ClusterName $clusterName `
-ResourceGroupName $resourceGroupName `
-HttpCredential $clusterCredential `
-Location $location `
-DefaultStorageAccountType "AzureDataLakeStore" `
-DefaultStorageAccountName adlsdatalakevmiedev.azuredatalakestore.net `
-DefaultStorageRootPath $storageRootPath `
-AadTenantId $tenantId `
-ObjectId $objectId `
-CertificateFilePath $certificateFilePath `
-CertificatePassword $password `
-ClusterSizeInNodes $clusterNodes `
-OSType $clusterOS `
-SshCredential $sshCredential `
-SshPublicKey $sshPublicKey `
-VirtualNetworkId $virtualNetworkId `
-SubnetName $subnet
When i am running this script it is throwing error and unable to use azure datalake store as default storage type.
error :
DEBUG: 9:30:34 PM - NewAzureHDInsightClusterCommand begin processing with ParameterSet 'CertificateFilePath'.
DEBUG: 9:30:34 PM - using account id 'MSI@50342'...
New-AzHDInsightCluster: Could not find file '/home/rakkula/System.Security.SecureString'.
DEBUG: AzureQoSEvent: CommandName - New-AzHDInsightCluster; IsSuccess - False; Duration - 00:00:00.0221324;; Exception - System.IO.FileNotFoundException: Could not find file '/home/rakkula/System.Security.SecureString'.
File name: '/home/rakkula/System.Security.SecureString'
at Interop.ThrowExceptionForIoErrno(ErrorInfo errorInfo, String path, Boolean isDirectory, Func`2 errorRewriter)
at Microsoft.Win32.SafeHandles.SafeFileHandle.Open(String path, OpenFlags flags, Int32 mode)
at System.IO.FileStream..ctor(String path, FileMode mode, FileAccess access, FileShare share, Int32 bufferSize, FileOptions options)
at System.IO.FileStream..ctor(String path, FileMode mode, FileAccess access, FileShare share, Int32 bufferSize)
at System.IO.File.ReadAllBytes(String path)
at Microsoft.Azure.Commands.HDInsight.NewAzureHDInsightClusterCommand.ExecuteCmdlet()
at Microsoft.WindowsAzure.Commands.Utilities.Common.AzurePSCmdlet.ProcessRecord();
DEBUG: Finish sending metric.
DEBUG: 9:30:34 PM - NewAzureHDInsightClusterCommand end processing.
PS /home/rakkula>
Could you please share any ex powershell script where i can pass metastore config for hive and oozie along . also am storing certificate in keyvault secrets and plz let me know how to refer it the script for alds access
