davidd2k · November 5, 2021 13:55 · Feb 5, 2018 · Feb 5, 2018 · Feb 5, 2018 · Feb 5, 2018
diff --git a/License.md b/License.md
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Darren Fuller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/ParquetDemo.csproj b/ParquetDemo.csproj
@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0" />
+    <PackageReference Include="Parquet.Net" Version="2.1.2" />
+    <PackageReference Include="System.Data.SqlClient" Version="4.4.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>
diff --git a/Program.cs b/Program.cs
@@ -0,0 +1,100 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SqlClient;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Configuration;
+using Parquet;
+using Parquet.Data;
+
+namespace ParquetDBDemo
+{
+    public static class Program
+    {
+        public static async Task Main(string[] args)
+        {
+            // Open the configuration file and read in the required configuration
+            var configBuilder = new ConfigurationBuilder().AddJsonFile("appsettings.json");
+            var configuration = configBuilder.Build();
+
+            var connectionString = configuration.GetConnectionString("local");
+            var outputFile = configuration["OutputFile"];
+
+            using (var connection = new SqlConnection(connectionString))
+            {
+                try
+                {
+                    await connection.OpenAsync();
+
+                    using (var command = connection.CreateCommand())
+                    {
+                        // Read in all the movie data
+                        command.CommandText = "SELECT * FROM dbo.movies";
+                        command.CommandType = CommandType.Text;
+
+                        using (var reader = await command.ExecuteReaderAsync())
+                        {
+                            // Get the column name and data type information for building up the parquet schema
+                            var columns = Enumerable.Range(0, reader.FieldCount)
+                                .Select(o => new {FieldName = reader.GetName(o), DataType = reader.GetFieldType(o)})
+                                .ToList();
+
+                            var schema = new List<DataField>(columns.Count);
+
+                            // Generate the Parquet schema from the column information, making sure that nullable types
+                            // are handled by making each field capable of holding null values
+                            schema.AddRange(from column in columns
+                                let columnType = column.DataType.IsClass
+                                    ? column.DataType
+                                    : typeof(Nullable<>).MakeGenericType(column.DataType)
+                                let fieldType = typeof(DataField<>).MakeGenericType(columnType)
+                                select (DataField) Activator.CreateInstance(fieldType, column.FieldName));
+
+                            var dataset = new Parquet.Data.DataSet(schema);
+                            var rowCount = 0;
+
+                            // Read each record from the database
+                            while (await reader.ReadAsync())
+                            {
+                                // Convert the current record into an object array, swapping DBNULL values for null before adding
+                                // the record to the parquet dataset
+                                var row = Enumerable.Range(0, reader.FieldCount)
+                                    .Select(o => reader.GetValue(o) == DBNull.Value ? null : reader.GetValue(o))
+                                    .ToArray();
+
+                                dataset.Add(row);
+
+                                rowCount++;
+                            }
+
+                            Console.WriteLine($"Records processed: {rowCount}");
+
+                            // Output the dataset to a parquet file on disk
+                            if (File.Exists(outputFile))
+                            {
+                                File.Delete(outputFile);
+                            }
+
+                            using (var writer = new ParquetWriter(File.OpenWrite(outputFile)))
+                            {
+                                writer.Write(dataset, CompressionMethod.Snappy);
+                            }
+                        }
+                    }
+                }
+                catch (SqlException e)
+                {
+                    Console.WriteLine(e);
+                    throw;
+                }
+                finally
+                {
+                    connection.Close();
+                }
+            }
+        }
+    }
+}
diff --git a/appsettings.json b/appsettings.json
@@ -0,0 +1,6 @@
+{
+  "OutputFile": "data.parquet", 
+  "ConnectionStrings": {
+    "local": "Server=<server name>;Initial Catalog=<database>;Integrated Security=SSPI"
+  }
+}
No results found