SQL Server 存储过程获取网站文本

Question

想知道这是否可能。

我需要编写一个 SQL Server 存储过程来从 Web API 页面获取一些文本。页面是https://bittrex.com/api/v1.1/public/getmarkets

如果只是将它收集为一个长字符串就很好了，我将使用 T-SQL 自己进行切割以将其放入不同的列中。

如果这是不可能的，有没有其他方法可以做到这一点？我宁愿不使用 SSIS，但欢迎使用最佳做法。

谢谢！

Answer 1

使用下面列出的三个辅助函数，还不错。

以下在 2.578 秒内返回 2,926 条记录（包括下载）。

如果您愿意，我应该添加 Parse 和 Parse-Row 函数可以内联

如果需要，您可以根据 RowNr 旋转以下内容。

示例

exec master..xp_cmdshell 'powershell.exe Invoke-WebRequest "https://bittrex.com/api/v1.1/public/getmarkets" -OutFile "c:\working\getmarkets.txt"',no_output

Declare @S varchar(max); 
Select @S = BulkColumn FROM  OPENROWSET(BULK 'c:\working\getmarkets.txt', SINGLE_BLOB) x; 

Select RowNr = A.RetSeq
      ,Item  = replace(C.Pos1,'"','')
      ,Value = replace(C.Pos2,'"','')
 From  [dbo].[tvf-Str-Extract] (@S,'{','}') A
 Cross Apply [dbo].[tvf-Str-Parse](A.RetVal,',') B
 Cross Apply [dbo].[tvf-Str-Parse-Row](B.RetVal,'":') C

退货

解析 UDF

CREATE FUNCTION [dbo].[tvf-Str-Parse] (@String varchar(max),@Delimiter varchar(10))
Returns Table 
As
Return (  
    Select RetSeq = Row_Number() over (Order By (Select null))
          ,RetVal = LTrim(RTrim(B.i.value('(./text())[1]', 'varchar(max)')))
    From  (Select x = Cast('<x>' + replace((Select replace(@String,@Delimiter,'§§Split§§') as [*] For XML Path('')),'§§Split§§','</x><x>')+'</x>' as xml).query('.')) as A 
    Cross Apply x.nodes('x') AS B(i)
);

提取 UDF

CREATE FUNCTION [dbo].[tvf-Str-Extract] (@String varchar(max),@Delimiter1 varchar(100),@Delimiter2 varchar(100))
Returns Table 
As
Return (  

with   cte1(N)   As (Select 1 From (Values(1),(1),(1),(1),(1),(1),(1),(1),(1),(1)) N(N)),
       cte2(N)   As (Select Top (IsNull(DataLength(@String),0)) Row_Number() over (Order By (Select NULL)) From (Select N=1 From cte1 N1,cte1 N2,cte1 N3,cte1 N4,cte1 N5,cte1 N6) A ),
       cte3(N)   As (Select 1 Union All Select t.N+DataLength(@Delimiter1) From cte2 t Where Substring(@String,t.N,DataLength(@Delimiter1)) = @Delimiter1),
       cte4(N,L) As (Select S.N,IsNull(NullIf(CharIndex(@Delimiter1,@String,s.N),0)-S.N,8000) From cte3 S)

Select RetSeq = Row_Number() over (Order By N)
      ,RetPos = N
      ,RetVal = left(RetVal,charindex(@Delimiter2,RetVal)-1) 
 From  (
        Select *,RetVal = Substring(@String, N, L) 
         From  cte4
       ) A
 Where charindex(@Delimiter2,RetVal)>1

)
/*
Max Length of String 1MM characters

Declare @String varchar(max) = 'Dear [[FirstName]] [[LastName]], ...'
Select * From [dbo].[tvf-Str-Extract] (@String,'[[',']]')
*/

最后是 Parse ROW UDF

CREATE FUNCTION [dbo].[tvf-Str-Parse-Row] (@String varchar(max),@Delimiter varchar(10))
Returns Table 
As
Return (
    Select Pos1 = ltrim(rtrim(xDim.value('/x[1]','varchar(max)')))
          ,Pos2 = ltrim(rtrim(xDim.value('/x[2]','varchar(max)')))
          ,Pos3 = ltrim(rtrim(xDim.value('/x[3]','varchar(max)')))
          ,Pos4 = ltrim(rtrim(xDim.value('/x[4]','varchar(max)')))
          ,Pos5 = ltrim(rtrim(xDim.value('/x[5]','varchar(max)')))
          ,Pos6 = ltrim(rtrim(xDim.value('/x[6]','varchar(max)')))
          ,Pos7 = ltrim(rtrim(xDim.value('/x[7]','varchar(max)')))
          ,Pos8 = ltrim(rtrim(xDim.value('/x[8]','varchar(max)')))
          ,Pos9 = ltrim(rtrim(xDim.value('/x[9]','varchar(max)')))
    From  (Select Cast('<x>' + replace((Select replace(@String,@Delimiter,'§§Split§§') as [*] For XML Path('')),'§§Split§§','</x><x>')+'</x>' as xml) as xDim) as A 
    Where @String is not null
)

编辑 - 只是为了好玩，我在 PIVOT 中折腾

Select *
 From (
        Select RowNr = A.RetSeq
              ,Item  = replace(C.Pos1,'"','')
              ,Value = replace(C.Pos2,'"','')
         From  [dbo].[tvf-Str-Extract] (@S,'{','}') A
         Cross Apply [dbo].[tvf-Str-Parse](A.RetVal,',') B
         Cross Apply [dbo].[tvf-Str-Parse-Row](B.RetVal,'":') C
      ) Src
 Pivot (max(value) for Item in ([MarketCurrency],[BaseCurrency],[MarketCurrencyLong],[BaseCurrencyLong],[MinTradeSize],[MarketName],[IsActive],[Created],[Notice],[IsSponsored],[LogoUrl])) p

退货