13 Kasım 2018

C# SqlBulkCopy ile büyük miktarda veri ekleme

Örnek olarak ekleyeceğimiz Customer tablosu


 CREATE TABLE [dbo].[Customer](
  Id bigint IDENTITY(1,1) NOT NULL,
  Name [nvarchar](60) NOT NULL,
  BirthDate [datetime] NOT NULL,
     Height INT NULL
     CONSTRAINT [PK_Customer] PRIMARY KEY CLUSTERED 
     (
    [ID] ASC
        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
     ) ON [PRIMARY]


C# İle Kullanımı


using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;

namespace SqlBulkCopyDemo
{
    public class Customer
    {
        public long Id { get; set; }
        public string Name { get; set; }
        public DateTime BirthDate { get; set; }
        public int Height { get; set; }

        public static IEnumerable<Customer> GenerateExampleData(int count)
        {
            int i = 0;
            while (i++ < count)
            {
                yield return new Customer()
                {
                    Name = Guid.NewGuid().ToString(),
                    BirthDate = DateTime.Today.AddDays(-1 * (i % (365 * 50))),
                    Height = new Random(i).Next(150, 190)
                };
            }

        }
    }

    class Program
    {
        static void Main(string[] args)
        {
            var customerCount = 5000000;
            var stopWatch = new Stopwatch();
            Console.WriteLine("Generating example data");
            stopWatch.Start();

            var dt = new DataTable();

            //Write Destination Column Names
            dt.TableName = "Customer";
            dt.Columns.Add("Name", typeof(string));
            dt.Columns.Add("BirthDate", typeof(DateTime));
            dt.Columns.Add("Height", typeof(int));

            foreach (var customer in Customer.GenerateExampleData(customerCount))
            {
                dt.Rows.Add(customer.Name, customer.BirthDate, customer.Height);
            }
            Console.WriteLine($"Data generated {stopWatch.Elapsed.ToString()}");

            Console.WriteLine($"SqlBulkCopy starting ");
            stopWatch.Restart();
            using (var con = new SqlConnection("Server=xxx;Database=xxx;User Id=xxx;Password=xxx;"))
            {

                using (var bulkCopy = new SqlBulkCopy(con, SqlBulkCopyOptions.TableLock, null))
                {
                    bulkCopy.DestinationTableName = "Customer";
                    bulkCopy.ColumnMappings.Add("Name", "Name");
                    bulkCopy.ColumnMappings.Add("BirthDate", "BirthDate");
                    bulkCopy.ColumnMappings.Add("Height", "Height");

                    bulkCopy.EnableStreaming = true;
                    bulkCopy.BatchSize = 10000;
                    bulkCopy.NotifyAfter = 1000;
                    bulkCopy.SqlRowsCopied += (sender, e) => Console.WriteLine("RowsCopied: " + e.RowsCopied);

                    var rows = new DataRow[dt.Rows.Count];
                    dt.Rows.CopyTo(rows, 0);

                    con.Open();
                    bulkCopy.WriteToServer(rows);
                }

            }
            stopWatch.Stop();
            Console.WriteLine($"Written to Database: {stopWatch.Elapsed.ToString()}");
            
            Console.ReadKey();
        }
    }
}